2024-11-22 03:39:40,995 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-11-22 03:39:41,009 main DEBUG Took 0.011293 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-22 03:39:41,009 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-22 03:39:41,010 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-22 03:39:41,010 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-22 03:39:41,012 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-22 03:39:41,022 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-22 03:39:41,040 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-22 03:39:41,041 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-22 03:39:41,042 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-22 03:39:41,043 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-22 03:39:41,044 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-22 03:39:41,044 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-22 03:39:41,046 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-22 03:39:41,046 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-22 03:39:41,047 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-22 03:39:41,048 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-22 03:39:41,049 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-22 03:39:41,050 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-22 03:39:41,051 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-22 03:39:41,051 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-22 03:39:41,052 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-22 03:39:41,053 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-22 03:39:41,053 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-22 03:39:41,054 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-22 03:39:41,055 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-22 03:39:41,055 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-22 03:39:41,056 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-22 03:39:41,057 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-22 03:39:41,058 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-22 03:39:41,058 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-22 03:39:41,059 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-22 03:39:41,059 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-22 03:39:41,062 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-22 03:39:41,064 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-22 03:39:41,067 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-22 03:39:41,068 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-22 03:39:41,070 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-22 03:39:41,072 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-22 03:39:41,086 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-22 03:39:41,091 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-22 03:39:41,093 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-22 03:39:41,094 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-22 03:39:41,095 main DEBUG createAppenders(={Console}) 2024-11-22 03:39:41,096 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad initialized 2024-11-22 03:39:41,097 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-11-22 03:39:41,097 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad OK. 2024-11-22 03:39:41,098 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-22 03:39:41,099 main DEBUG OutputStream closed 2024-11-22 03:39:41,099 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-22 03:39:41,099 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-22 03:39:41,100 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@55634720 OK 2024-11-22 03:39:41,209 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-22 03:39:41,212 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-22 03:39:41,214 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-22 03:39:41,216 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-22 03:39:41,217 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-22 03:39:41,217 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-22 03:39:41,218 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-22 03:39:41,218 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-22 03:39:41,218 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-22 03:39:41,219 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-22 03:39:41,219 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-22 03:39:41,220 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-22 03:39:41,220 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-22 03:39:41,221 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-22 03:39:41,221 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-22 03:39:41,221 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-22 03:39:41,222 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-22 03:39:41,223 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-22 03:39:41,226 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-22 03:39:41,227 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-logging/target/hbase-logging-4.0.0-alpha-1-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@19835e64) with optional ClassLoader: null 2024-11-22 03:39:41,227 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-22 03:39:41,228 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@19835e64] started OK. 2024-11-22T03:39:41,594 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ef424767-da8d-bd68-b32d-b1c48514e6de 2024-11-22 03:39:41,598 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-22 03:39:41,598 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-22T03:39:41,610 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.TestAcidGuaranteesWithNoInMemCompaction timeout: 13 mins 2024-11-22T03:39:41,638 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=1, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-22T03:39:41,657 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ef424767-da8d-bd68-b32d-b1c48514e6de/cluster_b5649d9b-69fe-21c7-9e96-7cb2331ed67f, deleteOnExit=true 2024-11-22T03:39:41,658 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-11-22T03:39:41,659 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ef424767-da8d-bd68-b32d-b1c48514e6de/test.cache.data in system properties and HBase conf 2024-11-22T03:39:41,660 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ef424767-da8d-bd68-b32d-b1c48514e6de/hadoop.tmp.dir in system properties and HBase conf 2024-11-22T03:39:41,663 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ef424767-da8d-bd68-b32d-b1c48514e6de/hadoop.log.dir in system properties and HBase conf 2024-11-22T03:39:41,664 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ef424767-da8d-bd68-b32d-b1c48514e6de/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-22T03:39:41,665 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ef424767-da8d-bd68-b32d-b1c48514e6de/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-22T03:39:41,665 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-22T03:39:41,778 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-22T03:39:41,935 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-22T03:39:41,944 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ef424767-da8d-bd68-b32d-b1c48514e6de/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-22T03:39:41,945 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ef424767-da8d-bd68-b32d-b1c48514e6de/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-22T03:39:41,948 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ef424767-da8d-bd68-b32d-b1c48514e6de/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-22T03:39:41,948 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ef424767-da8d-bd68-b32d-b1c48514e6de/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-22T03:39:41,949 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ef424767-da8d-bd68-b32d-b1c48514e6de/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-22T03:39:41,951 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ef424767-da8d-bd68-b32d-b1c48514e6de/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-22T03:39:41,955 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ef424767-da8d-bd68-b32d-b1c48514e6de/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-22T03:39:41,956 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ef424767-da8d-bd68-b32d-b1c48514e6de/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-22T03:39:41,957 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ef424767-da8d-bd68-b32d-b1c48514e6de/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-22T03:39:41,958 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ef424767-da8d-bd68-b32d-b1c48514e6de/nfs.dump.dir in system properties and HBase conf 2024-11-22T03:39:41,959 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ef424767-da8d-bd68-b32d-b1c48514e6de/java.io.tmpdir in system properties and HBase conf 2024-11-22T03:39:41,959 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ef424767-da8d-bd68-b32d-b1c48514e6de/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-22T03:39:41,960 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ef424767-da8d-bd68-b32d-b1c48514e6de/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-22T03:39:41,961 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ef424767-da8d-bd68-b32d-b1c48514e6de/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-22T03:39:42,956 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-11-22T03:39:43,055 INFO [Time-limited test {}] log.Log(170): Logging initialized @2910ms to org.eclipse.jetty.util.log.Slf4jLog 2024-11-22T03:39:43,141 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-22T03:39:43,207 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-22T03:39:43,227 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-22T03:39:43,227 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-22T03:39:43,229 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-22T03:39:43,241 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-22T03:39:43,244 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6b23711f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ef424767-da8d-bd68-b32d-b1c48514e6de/hadoop.log.dir/,AVAILABLE} 2024-11-22T03:39:43,245 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@93c62df{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-22T03:39:43,462 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4ded99d3{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ef424767-da8d-bd68-b32d-b1c48514e6de/java.io.tmpdir/jetty-localhost-39585-hadoop-hdfs-3_4_1-tests_jar-_-any-9356338785202672745/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-22T03:39:43,469 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@557285fa{HTTP/1.1, (http/1.1)}{localhost:39585} 2024-11-22T03:39:43,469 INFO [Time-limited test {}] server.Server(415): Started @3325ms 2024-11-22T03:39:43,866 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-22T03:39:43,875 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-22T03:39:43,876 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-22T03:39:43,876 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-22T03:39:43,877 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-22T03:39:43,877 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@4fd7fa7c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ef424767-da8d-bd68-b32d-b1c48514e6de/hadoop.log.dir/,AVAILABLE} 2024-11-22T03:39:43,878 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@435f9f96{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-22T03:39:43,983 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4269bd31{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ef424767-da8d-bd68-b32d-b1c48514e6de/java.io.tmpdir/jetty-localhost-39723-hadoop-hdfs-3_4_1-tests_jar-_-any-10000838027952485795/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-22T03:39:43,984 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@353d64f2{HTTP/1.1, (http/1.1)}{localhost:39723} 2024-11-22T03:39:43,984 INFO [Time-limited test {}] server.Server(415): Started @3840ms 2024-11-22T03:39:44,038 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-22T03:39:44,590 WARN [Thread-72 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ef424767-da8d-bd68-b32d-b1c48514e6de/cluster_b5649d9b-69fe-21c7-9e96-7cb2331ed67f/data/data1/current/BP-418171142-172.17.0.2-1732246782681/current, will proceed with Du for space computation calculation, 2024-11-22T03:39:44,597 WARN [Thread-73 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ef424767-da8d-bd68-b32d-b1c48514e6de/cluster_b5649d9b-69fe-21c7-9e96-7cb2331ed67f/data/data2/current/BP-418171142-172.17.0.2-1732246782681/current, will proceed with Du for space computation calculation, 2024-11-22T03:39:44,704 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-22T03:39:44,771 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x612d7786b63146f1 with lease ID 0x14669bf9256dc031: Processing first storage report for DS-89103e69-741c-4fec-ac01-a162c6d86ae3 from datanode DatanodeRegistration(127.0.0.1:41449, datanodeUuid=1704e707-0098-49a6-9504-b28a4f653dc9, infoPort=38805, infoSecurePort=0, ipcPort=45997, storageInfo=lv=-57;cid=testClusterID;nsid=1881469965;c=1732246782681) 2024-11-22T03:39:44,774 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x612d7786b63146f1 with lease ID 0x14669bf9256dc031: from storage DS-89103e69-741c-4fec-ac01-a162c6d86ae3 node DatanodeRegistration(127.0.0.1:41449, datanodeUuid=1704e707-0098-49a6-9504-b28a4f653dc9, infoPort=38805, infoSecurePort=0, ipcPort=45997, storageInfo=lv=-57;cid=testClusterID;nsid=1881469965;c=1732246782681), blocks: 0, hasStaleStorage: true, processing time: 3 msecs, invalidatedBlocks: 0 2024-11-22T03:39:44,775 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x612d7786b63146f1 with lease ID 0x14669bf9256dc031: Processing first storage report for DS-73f43a8e-4b6b-4707-8938-79efcfa96ed3 from datanode DatanodeRegistration(127.0.0.1:41449, datanodeUuid=1704e707-0098-49a6-9504-b28a4f653dc9, infoPort=38805, infoSecurePort=0, ipcPort=45997, storageInfo=lv=-57;cid=testClusterID;nsid=1881469965;c=1732246782681) 2024-11-22T03:39:44,775 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x612d7786b63146f1 with lease ID 0x14669bf9256dc031: from storage DS-73f43a8e-4b6b-4707-8938-79efcfa96ed3 node DatanodeRegistration(127.0.0.1:41449, datanodeUuid=1704e707-0098-49a6-9504-b28a4f653dc9, infoPort=38805, infoSecurePort=0, ipcPort=45997, storageInfo=lv=-57;cid=testClusterID;nsid=1881469965;c=1732246782681), blocks: 0, hasStaleStorage: false, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-22T03:39:44,822 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ef424767-da8d-bd68-b32d-b1c48514e6de 2024-11-22T03:39:44,924 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ef424767-da8d-bd68-b32d-b1c48514e6de/cluster_b5649d9b-69fe-21c7-9e96-7cb2331ed67f/zookeeper_0, clientPort=64888, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ef424767-da8d-bd68-b32d-b1c48514e6de/cluster_b5649d9b-69fe-21c7-9e96-7cb2331ed67f/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ef424767-da8d-bd68-b32d-b1c48514e6de/cluster_b5649d9b-69fe-21c7-9e96-7cb2331ed67f/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-22T03:39:44,937 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=64888 2024-11-22T03:39:44,953 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-22T03:39:44,958 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-22T03:39:45,253 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073741825_1001 (size=7) 2024-11-22T03:39:45,659 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a with version=8 2024-11-22T03:39:45,659 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/hbase-staging 2024-11-22T03:39:45,747 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-22T03:39:45,956 INFO [Time-limited test {}] client.ConnectionUtils(128): master/c85114ed5096:0 server-side Connection retries=45 2024-11-22T03:39:45,965 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-22T03:39:45,966 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-22T03:39:45,972 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-22T03:39:45,972 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-22T03:39:45,972 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-22T03:39:46,124 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-22T03:39:46,193 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-22T03:39:46,205 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-22T03:39:46,210 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-22T03:39:46,237 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 1533 (auto-detected) 2024-11-22T03:39:46,238 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-11-22T03:39:46,257 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:42627 2024-11-22T03:39:46,281 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:42627 connecting to ZooKeeper ensemble=127.0.0.1:64888 2024-11-22T03:39:46,311 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:426270x0, quorum=127.0.0.1:64888, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-22T03:39:46,315 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:42627-0x1006583a7f00000 connected 2024-11-22T03:39:46,349 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-22T03:39:46,351 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-22T03:39:46,360 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:42627-0x1006583a7f00000, quorum=127.0.0.1:64888, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-22T03:39:46,364 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a, hbase.cluster.distributed=false 2024-11-22T03:39:46,389 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:42627-0x1006583a7f00000, quorum=127.0.0.1:64888, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-22T03:39:46,394 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=42627 2024-11-22T03:39:46,397 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=42627 2024-11-22T03:39:46,398 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=42627 2024-11-22T03:39:46,398 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=42627 2024-11-22T03:39:46,399 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=42627 2024-11-22T03:39:46,530 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/c85114ed5096:0 server-side Connection retries=45 2024-11-22T03:39:46,531 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-22T03:39:46,532 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-22T03:39:46,532 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-22T03:39:46,532 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-22T03:39:46,532 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-22T03:39:46,536 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-22T03:39:46,539 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-22T03:39:46,540 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:42545 2024-11-22T03:39:46,543 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:42545 connecting to ZooKeeper ensemble=127.0.0.1:64888 2024-11-22T03:39:46,544 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-22T03:39:46,548 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-22T03:39:46,555 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:425450x0, quorum=127.0.0.1:64888, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-22T03:39:46,556 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:425450x0, quorum=127.0.0.1:64888, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-22T03:39:46,556 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:42545-0x1006583a7f00001 connected 2024-11-22T03:39:46,560 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-22T03:39:46,571 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-22T03:39:46,574 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:42545-0x1006583a7f00001, quorum=127.0.0.1:64888, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-22T03:39:46,579 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:42545-0x1006583a7f00001, quorum=127.0.0.1:64888, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-22T03:39:46,580 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=42545 2024-11-22T03:39:46,581 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=42545 2024-11-22T03:39:46,581 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=42545 2024-11-22T03:39:46,583 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=42545 2024-11-22T03:39:46,584 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=42545 2024-11-22T03:39:46,600 DEBUG [M:0;c85114ed5096:42627 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;c85114ed5096:42627 2024-11-22T03:39:46,601 INFO [master/c85114ed5096:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/c85114ed5096,42627,1732246785796 2024-11-22T03:39:46,607 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42545-0x1006583a7f00001, quorum=127.0.0.1:64888, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-22T03:39:46,607 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42627-0x1006583a7f00000, quorum=127.0.0.1:64888, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-22T03:39:46,608 DEBUG [master/c85114ed5096:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:42627-0x1006583a7f00000, quorum=127.0.0.1:64888, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/c85114ed5096,42627,1732246785796 2024-11-22T03:39:46,634 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42627-0x1006583a7f00000, quorum=127.0.0.1:64888, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-22T03:39:46,635 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42545-0x1006583a7f00001, quorum=127.0.0.1:64888, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-22T03:39:46,635 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42545-0x1006583a7f00001, quorum=127.0.0.1:64888, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-22T03:39:46,635 DEBUG [master/c85114ed5096:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:42627-0x1006583a7f00000, quorum=127.0.0.1:64888, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-22T03:39:46,636 INFO [master/c85114ed5096:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/c85114ed5096,42627,1732246785796 from backup master directory 2024-11-22T03:39:46,639 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42627-0x1006583a7f00000, quorum=127.0.0.1:64888, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/c85114ed5096,42627,1732246785796 2024-11-22T03:39:46,639 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42545-0x1006583a7f00001, quorum=127.0.0.1:64888, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-22T03:39:46,639 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42627-0x1006583a7f00000, quorum=127.0.0.1:64888, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-22T03:39:46,640 WARN [master/c85114ed5096:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-22T03:39:46,640 INFO [master/c85114ed5096:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=c85114ed5096,42627,1732246785796 2024-11-22T03:39:46,643 INFO [master/c85114ed5096:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-22T03:39:46,644 INFO [master/c85114ed5096:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-22T03:39:46,710 DEBUG [master/c85114ed5096:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/hbase.id] with ID: 67035b7f-c649-44a8-9db7-17adff49cedb 2024-11-22T03:39:46,710 DEBUG [master/c85114ed5096:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/.tmp/hbase.id 2024-11-22T03:39:46,721 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073741826_1002 (size=42) 2024-11-22T03:39:47,123 DEBUG [master/c85114ed5096:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/.tmp/hbase.id]:[hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/hbase.id] 2024-11-22T03:39:47,168 INFO [master/c85114ed5096:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-22T03:39:47,173 INFO [master/c85114ed5096:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-22T03:39:47,195 INFO [master/c85114ed5096:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 20ms. 2024-11-22T03:39:47,199 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42545-0x1006583a7f00001, quorum=127.0.0.1:64888, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-22T03:39:47,200 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42627-0x1006583a7f00000, quorum=127.0.0.1:64888, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-22T03:39:47,214 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073741827_1003 (size=196) 2024-11-22T03:39:47,636 INFO [master/c85114ed5096:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-22T03:39:47,639 INFO [master/c85114ed5096:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-22T03:39:47,671 DEBUG [master/c85114ed5096:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:39:47,677 INFO [master/c85114ed5096:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-22T03:39:47,712 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073741828_1004 (size=1189) 2024-11-22T03:39:48,139 INFO [master/c85114ed5096:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/MasterData/data/master/store 2024-11-22T03:39:48,160 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073741829_1005 (size=34) 2024-11-22T03:39:48,567 INFO [master/c85114ed5096:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-22T03:39:48,571 DEBUG [master/c85114ed5096:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-22T03:39:48,573 DEBUG [master/c85114ed5096:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-22T03:39:48,573 INFO [master/c85114ed5096:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-22T03:39:48,574 DEBUG [master/c85114ed5096:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-22T03:39:48,576 DEBUG [master/c85114ed5096:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-22T03:39:48,577 DEBUG [master/c85114ed5096:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-22T03:39:48,577 INFO [master/c85114ed5096:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-22T03:39:48,579 DEBUG [master/c85114ed5096:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732246788573Disabling compacts and flushes for region at 1732246788573Disabling writes for close at 1732246788576 (+3 ms)Writing region close event to WAL at 1732246788577 (+1 ms)Closed at 1732246788577 2024-11-22T03:39:48,581 WARN [master/c85114ed5096:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/MasterData/data/master/store/.initializing 2024-11-22T03:39:48,582 DEBUG [master/c85114ed5096:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/MasterData/WALs/c85114ed5096,42627,1732246785796 2024-11-22T03:39:48,594 INFO [master/c85114ed5096:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-22T03:39:48,611 INFO [master/c85114ed5096:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=c85114ed5096%2C42627%2C1732246785796, suffix=, logDir=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/MasterData/WALs/c85114ed5096,42627,1732246785796, archiveDir=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/MasterData/oldWALs, maxLogs=10 2024-11-22T03:39:48,643 DEBUG [master/c85114ed5096:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/MasterData/WALs/c85114ed5096,42627,1732246785796/c85114ed5096%2C42627%2C1732246785796.1732246788616, exclude list is [], retry=0 2024-11-22T03:39:48,662 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41449,DS-89103e69-741c-4fec-ac01-a162c6d86ae3,DISK] 2024-11-22T03:39:48,665 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-11-22T03:39:48,703 INFO [master/c85114ed5096:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/MasterData/WALs/c85114ed5096,42627,1732246785796/c85114ed5096%2C42627%2C1732246785796.1732246788616 2024-11-22T03:39:48,705 DEBUG [master/c85114ed5096:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:38805:38805)] 2024-11-22T03:39:48,706 DEBUG [master/c85114ed5096:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-22T03:39:48,706 DEBUG [master/c85114ed5096:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-22T03:39:48,710 DEBUG [master/c85114ed5096:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-22T03:39:48,711 DEBUG [master/c85114ed5096:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-22T03:39:48,770 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-22T03:39:48,811 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-22T03:39:48,814 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:39:48,816 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-22T03:39:48,817 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-22T03:39:48,820 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-22T03:39:48,820 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:39:48,822 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-22T03:39:48,822 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-22T03:39:48,825 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-22T03:39:48,825 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:39:48,826 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-22T03:39:48,826 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-22T03:39:48,829 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-22T03:39:48,829 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:39:48,830 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-22T03:39:48,831 DEBUG [master/c85114ed5096:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-22T03:39:48,834 DEBUG [master/c85114ed5096:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-22T03:39:48,835 DEBUG [master/c85114ed5096:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-22T03:39:48,841 DEBUG [master/c85114ed5096:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-22T03:39:48,841 DEBUG [master/c85114ed5096:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-22T03:39:48,845 DEBUG [master/c85114ed5096:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-22T03:39:48,848 DEBUG [master/c85114ed5096:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-22T03:39:48,853 DEBUG [master/c85114ed5096:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-22T03:39:48,854 INFO [master/c85114ed5096:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60513443, jitterRate=-0.0982794314622879}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-22T03:39:48,861 DEBUG [master/c85114ed5096:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1732246788728Initializing all the Stores at 1732246788731 (+3 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732246788732 (+1 ms)Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732246788737 (+5 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732246788743 (+6 ms)Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732246788743Cleaning up temporary data from old regions at 1732246788841 (+98 ms)Region opened successfully at 1732246788861 (+20 ms) 2024-11-22T03:39:48,863 INFO [master/c85114ed5096:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-22T03:39:48,897 DEBUG [master/c85114ed5096:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@77d58de, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=c85114ed5096/172.17.0.2:0 2024-11-22T03:39:48,929 INFO [master/c85114ed5096:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-22T03:39:48,938 INFO [master/c85114ed5096:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-22T03:39:48,938 INFO [master/c85114ed5096:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-22T03:39:48,941 INFO [master/c85114ed5096:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-22T03:39:48,942 INFO [master/c85114ed5096:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-22T03:39:48,947 INFO [master/c85114ed5096:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-11-22T03:39:48,947 INFO [master/c85114ed5096:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-22T03:39:48,976 INFO [master/c85114ed5096:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-22T03:39:48,984 DEBUG [master/c85114ed5096:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42627-0x1006583a7f00000, quorum=127.0.0.1:64888, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-22T03:39:48,986 DEBUG [master/c85114ed5096:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-22T03:39:48,989 INFO [master/c85114ed5096:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-22T03:39:48,990 DEBUG [master/c85114ed5096:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42627-0x1006583a7f00000, quorum=127.0.0.1:64888, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-22T03:39:48,991 DEBUG [master/c85114ed5096:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-22T03:39:48,993 INFO [master/c85114ed5096:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-22T03:39:48,997 DEBUG [master/c85114ed5096:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42627-0x1006583a7f00000, quorum=127.0.0.1:64888, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-22T03:39:48,998 DEBUG [master/c85114ed5096:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-22T03:39:48,999 DEBUG [master/c85114ed5096:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42627-0x1006583a7f00000, quorum=127.0.0.1:64888, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-22T03:39:49,000 DEBUG [master/c85114ed5096:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-22T03:39:49,016 DEBUG [master/c85114ed5096:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42627-0x1006583a7f00000, quorum=127.0.0.1:64888, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-22T03:39:49,018 DEBUG [master/c85114ed5096:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-22T03:39:49,021 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42545-0x1006583a7f00001, quorum=127.0.0.1:64888, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-22T03:39:49,021 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42627-0x1006583a7f00000, quorum=127.0.0.1:64888, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-22T03:39:49,021 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42627-0x1006583a7f00000, quorum=127.0.0.1:64888, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-22T03:39:49,021 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42545-0x1006583a7f00001, quorum=127.0.0.1:64888, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-22T03:39:49,023 INFO [master/c85114ed5096:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=c85114ed5096,42627,1732246785796, sessionid=0x1006583a7f00000, setting cluster-up flag (Was=false) 2024-11-22T03:39:49,034 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42545-0x1006583a7f00001, quorum=127.0.0.1:64888, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-22T03:39:49,034 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42627-0x1006583a7f00000, quorum=127.0.0.1:64888, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-22T03:39:49,038 DEBUG [master/c85114ed5096:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-22T03:39:49,039 DEBUG [master/c85114ed5096:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=c85114ed5096,42627,1732246785796 2024-11-22T03:39:49,043 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42627-0x1006583a7f00000, quorum=127.0.0.1:64888, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-22T03:39:49,043 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42545-0x1006583a7f00001, quorum=127.0.0.1:64888, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-22T03:39:49,048 DEBUG [master/c85114ed5096:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-22T03:39:49,050 DEBUG [master/c85114ed5096:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=c85114ed5096,42627,1732246785796 2024-11-22T03:39:49,056 INFO [master/c85114ed5096:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-22T03:39:49,091 INFO [RS:0;c85114ed5096:42545 {}] regionserver.HRegionServer(746): ClusterId : 67035b7f-c649-44a8-9db7-17adff49cedb 2024-11-22T03:39:49,093 DEBUG [RS:0;c85114ed5096:42545 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-22T03:39:49,098 DEBUG [RS:0;c85114ed5096:42545 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-22T03:39:49,098 DEBUG [RS:0;c85114ed5096:42545 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-22T03:39:49,101 DEBUG [RS:0;c85114ed5096:42545 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-22T03:39:49,101 DEBUG [RS:0;c85114ed5096:42545 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@352a475e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=c85114ed5096/172.17.0.2:0 2024-11-22T03:39:49,114 DEBUG [RS:0;c85114ed5096:42545 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;c85114ed5096:42545 2024-11-22T03:39:49,118 INFO [RS:0;c85114ed5096:42545 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-22T03:39:49,118 INFO [RS:0;c85114ed5096:42545 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-22T03:39:49,118 DEBUG [RS:0;c85114ed5096:42545 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-22T03:39:49,121 INFO [RS:0;c85114ed5096:42545 {}] regionserver.HRegionServer(2659): reportForDuty to master=c85114ed5096,42627,1732246785796 with port=42545, startcode=1732246786482 2024-11-22T03:39:49,132 DEBUG [master/c85114ed5096:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-22T03:39:49,134 DEBUG [RS:0;c85114ed5096:42545 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-22T03:39:49,146 INFO [master/c85114ed5096:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-22T03:39:49,155 INFO [master/c85114ed5096:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-22T03:39:49,162 DEBUG [master/c85114ed5096:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: c85114ed5096,42627,1732246785796 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-22T03:39:49,171 DEBUG [master/c85114ed5096:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/c85114ed5096:0, corePoolSize=5, maxPoolSize=5 2024-11-22T03:39:49,172 DEBUG [master/c85114ed5096:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/c85114ed5096:0, corePoolSize=5, maxPoolSize=5 2024-11-22T03:39:49,172 DEBUG [master/c85114ed5096:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/c85114ed5096:0, corePoolSize=5, maxPoolSize=5 2024-11-22T03:39:49,172 DEBUG [master/c85114ed5096:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/c85114ed5096:0, corePoolSize=5, maxPoolSize=5 2024-11-22T03:39:49,172 DEBUG [master/c85114ed5096:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/c85114ed5096:0, corePoolSize=10, maxPoolSize=10 2024-11-22T03:39:49,173 DEBUG [master/c85114ed5096:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/c85114ed5096:0, corePoolSize=1, maxPoolSize=1 2024-11-22T03:39:49,173 DEBUG [master/c85114ed5096:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/c85114ed5096:0, corePoolSize=2, maxPoolSize=2 2024-11-22T03:39:49,173 DEBUG [master/c85114ed5096:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/c85114ed5096:0, corePoolSize=1, maxPoolSize=1 2024-11-22T03:39:49,176 INFO [master/c85114ed5096:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732246819176 2024-11-22T03:39:49,177 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59913, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-11-22T03:39:49,178 INFO [master/c85114ed5096:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-22T03:39:49,180 INFO [master/c85114ed5096:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-22T03:39:49,181 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-22T03:39:49,181 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-22T03:39:49,183 INFO [master/c85114ed5096:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-22T03:39:49,184 INFO [master/c85114ed5096:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-22T03:39:49,184 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42627 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:667) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16714) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:39:49,184 INFO [master/c85114ed5096:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-22T03:39:49,184 INFO [master/c85114ed5096:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-22T03:39:49,187 INFO [master/c85114ed5096:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-22T03:39:49,191 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:39:49,192 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-22T03:39:49,197 INFO [master/c85114ed5096:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-22T03:39:49,199 INFO [master/c85114ed5096:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-22T03:39:49,200 INFO [master/c85114ed5096:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-22T03:39:49,203 INFO [master/c85114ed5096:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-22T03:39:49,203 INFO [master/c85114ed5096:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-22T03:39:49,210 DEBUG [master/c85114ed5096:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/c85114ed5096:0:becomeActiveMaster-HFileCleaner.large.0-1732246789205,5,FailOnTimeoutGroup] 2024-11-22T03:39:49,211 DEBUG [master/c85114ed5096:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/c85114ed5096:0:becomeActiveMaster-HFileCleaner.small.0-1732246789210,5,FailOnTimeoutGroup] 2024-11-22T03:39:49,211 INFO [master/c85114ed5096:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-22T03:39:49,211 INFO [master/c85114ed5096:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-22T03:39:49,212 INFO [master/c85114ed5096:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-22T03:39:49,213 INFO [master/c85114ed5096:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-22T03:39:49,215 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073741831_1007 (size=1321) 2024-11-22T03:39:49,226 DEBUG [RS:0;c85114ed5096:42545 {}] regionserver.HRegionServer(2683): Master is not running yet 2024-11-22T03:39:49,226 WARN [RS:0;c85114ed5096:42545 {}] regionserver.HRegionServer(841): reportForDuty failed; sleeping 100 ms and then retrying. 2024-11-22T03:39:49,328 INFO [RS:0;c85114ed5096:42545 {}] regionserver.HRegionServer(2659): reportForDuty to master=c85114ed5096,42627,1732246785796 with port=42545, startcode=1732246786482 2024-11-22T03:39:49,331 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42627 {}] master.ServerManager(363): Checking decommissioned status of RegionServer c85114ed5096,42545,1732246786482 2024-11-22T03:39:49,333 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42627 {}] master.ServerManager(517): Registering regionserver=c85114ed5096,42545,1732246786482 2024-11-22T03:39:49,343 DEBUG [RS:0;c85114ed5096:42545 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a 2024-11-22T03:39:49,343 DEBUG [RS:0;c85114ed5096:42545 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:35439 2024-11-22T03:39:49,343 DEBUG [RS:0;c85114ed5096:42545 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-22T03:39:49,347 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42627-0x1006583a7f00000, quorum=127.0.0.1:64888, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-22T03:39:49,347 DEBUG [RS:0;c85114ed5096:42545 {}] zookeeper.ZKUtil(111): regionserver:42545-0x1006583a7f00001, quorum=127.0.0.1:64888, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/c85114ed5096,42545,1732246786482 2024-11-22T03:39:49,348 WARN [RS:0;c85114ed5096:42545 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-22T03:39:49,348 INFO [RS:0;c85114ed5096:42545 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-22T03:39:49,348 DEBUG [RS:0;c85114ed5096:42545 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/WALs/c85114ed5096,42545,1732246786482 2024-11-22T03:39:49,350 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [c85114ed5096,42545,1732246786482] 2024-11-22T03:39:49,373 INFO [RS:0;c85114ed5096:42545 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-22T03:39:49,391 INFO [RS:0;c85114ed5096:42545 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-22T03:39:49,396 INFO [RS:0;c85114ed5096:42545 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-22T03:39:49,397 INFO [RS:0;c85114ed5096:42545 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-22T03:39:49,398 INFO [RS:0;c85114ed5096:42545 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-22T03:39:49,403 INFO [RS:0;c85114ed5096:42545 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-22T03:39:49,405 INFO [RS:0;c85114ed5096:42545 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-22T03:39:49,405 DEBUG [RS:0;c85114ed5096:42545 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/c85114ed5096:0, corePoolSize=1, maxPoolSize=1 2024-11-22T03:39:49,405 DEBUG [RS:0;c85114ed5096:42545 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/c85114ed5096:0, corePoolSize=1, maxPoolSize=1 2024-11-22T03:39:49,405 DEBUG [RS:0;c85114ed5096:42545 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/c85114ed5096:0, corePoolSize=1, maxPoolSize=1 2024-11-22T03:39:49,406 DEBUG [RS:0;c85114ed5096:42545 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/c85114ed5096:0, corePoolSize=1, maxPoolSize=1 2024-11-22T03:39:49,406 DEBUG [RS:0;c85114ed5096:42545 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/c85114ed5096:0, corePoolSize=1, maxPoolSize=1 2024-11-22T03:39:49,406 DEBUG [RS:0;c85114ed5096:42545 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/c85114ed5096:0, corePoolSize=2, maxPoolSize=2 2024-11-22T03:39:49,406 DEBUG [RS:0;c85114ed5096:42545 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/c85114ed5096:0, corePoolSize=1, maxPoolSize=1 2024-11-22T03:39:49,406 DEBUG [RS:0;c85114ed5096:42545 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/c85114ed5096:0, corePoolSize=1, maxPoolSize=1 2024-11-22T03:39:49,406 DEBUG [RS:0;c85114ed5096:42545 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/c85114ed5096:0, corePoolSize=1, maxPoolSize=1 2024-11-22T03:39:49,407 DEBUG [RS:0;c85114ed5096:42545 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/c85114ed5096:0, corePoolSize=1, maxPoolSize=1 2024-11-22T03:39:49,407 DEBUG [RS:0;c85114ed5096:42545 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/c85114ed5096:0, corePoolSize=1, maxPoolSize=1 2024-11-22T03:39:49,407 DEBUG [RS:0;c85114ed5096:42545 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/c85114ed5096:0, corePoolSize=1, maxPoolSize=1 2024-11-22T03:39:49,407 DEBUG [RS:0;c85114ed5096:42545 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/c85114ed5096:0, corePoolSize=3, maxPoolSize=3 2024-11-22T03:39:49,408 DEBUG [RS:0;c85114ed5096:42545 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0, corePoolSize=3, maxPoolSize=3 2024-11-22T03:39:49,409 INFO [RS:0;c85114ed5096:42545 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-22T03:39:49,409 INFO [RS:0;c85114ed5096:42545 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-22T03:39:49,410 INFO [RS:0;c85114ed5096:42545 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-22T03:39:49,410 INFO [RS:0;c85114ed5096:42545 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-22T03:39:49,410 INFO [RS:0;c85114ed5096:42545 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-22T03:39:49,410 INFO [RS:0;c85114ed5096:42545 {}] hbase.ChoreService(168): Chore ScheduledChore name=c85114ed5096,42545,1732246786482-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-22T03:39:49,430 INFO [RS:0;c85114ed5096:42545 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-22T03:39:49,432 INFO [RS:0;c85114ed5096:42545 {}] hbase.ChoreService(168): Chore ScheduledChore name=c85114ed5096,42545,1732246786482-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-22T03:39:49,433 INFO [RS:0;c85114ed5096:42545 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-22T03:39:49,433 INFO [RS:0;c85114ed5096:42545 {}] regionserver.Replication(171): c85114ed5096,42545,1732246786482 started 2024-11-22T03:39:49,452 INFO [RS:0;c85114ed5096:42545 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-22T03:39:49,452 INFO [RS:0;c85114ed5096:42545 {}] regionserver.HRegionServer(1482): Serving as c85114ed5096,42545,1732246786482, RpcServer on c85114ed5096/172.17.0.2:42545, sessionid=0x1006583a7f00001 2024-11-22T03:39:49,453 DEBUG [RS:0;c85114ed5096:42545 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-22T03:39:49,453 DEBUG [RS:0;c85114ed5096:42545 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager c85114ed5096,42545,1732246786482 2024-11-22T03:39:49,453 DEBUG [RS:0;c85114ed5096:42545 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'c85114ed5096,42545,1732246786482' 2024-11-22T03:39:49,453 DEBUG [RS:0;c85114ed5096:42545 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-22T03:39:49,454 DEBUG [RS:0;c85114ed5096:42545 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-22T03:39:49,455 DEBUG [RS:0;c85114ed5096:42545 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-22T03:39:49,455 DEBUG [RS:0;c85114ed5096:42545 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-22T03:39:49,455 DEBUG [RS:0;c85114ed5096:42545 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager c85114ed5096,42545,1732246786482 2024-11-22T03:39:49,456 DEBUG [RS:0;c85114ed5096:42545 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'c85114ed5096,42545,1732246786482' 2024-11-22T03:39:49,456 DEBUG [RS:0;c85114ed5096:42545 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-22T03:39:49,456 DEBUG [RS:0;c85114ed5096:42545 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-22T03:39:49,457 DEBUG [RS:0;c85114ed5096:42545 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-22T03:39:49,457 INFO [RS:0;c85114ed5096:42545 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-22T03:39:49,457 INFO [RS:0;c85114ed5096:42545 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-22T03:39:49,569 INFO [RS:0;c85114ed5096:42545 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-22T03:39:49,572 INFO [RS:0;c85114ed5096:42545 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=c85114ed5096%2C42545%2C1732246786482, suffix=, logDir=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/WALs/c85114ed5096,42545,1732246786482, archiveDir=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/oldWALs, maxLogs=32 2024-11-22T03:39:49,588 DEBUG [RS:0;c85114ed5096:42545 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/WALs/c85114ed5096,42545,1732246786482/c85114ed5096%2C42545%2C1732246786482.1732246789575, exclude list is [], retry=0 2024-11-22T03:39:49,592 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41449,DS-89103e69-741c-4fec-ac01-a162c6d86ae3,DISK] 2024-11-22T03:39:49,596 INFO [RS:0;c85114ed5096:42545 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/WALs/c85114ed5096,42545,1732246786482/c85114ed5096%2C42545%2C1732246786482.1732246789575 2024-11-22T03:39:49,597 DEBUG [RS:0;c85114ed5096:42545 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:38805:38805)] 2024-11-22T03:39:49,617 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-22T03:39:49,618 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a 2024-11-22T03:39:49,629 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073741833_1009 (size=32) 2024-11-22T03:39:50,033 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-22T03:39:50,037 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-22T03:39:50,041 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-22T03:39:50,041 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:39:50,042 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-22T03:39:50,043 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-22T03:39:50,045 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-22T03:39:50,045 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:39:50,046 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-22T03:39:50,046 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-22T03:39:50,048 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-22T03:39:50,049 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:39:50,050 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-22T03:39:50,050 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-22T03:39:50,053 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-22T03:39:50,053 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:39:50,054 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-22T03:39:50,054 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-22T03:39:50,056 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/hbase/meta/1588230740 2024-11-22T03:39:50,057 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/hbase/meta/1588230740 2024-11-22T03:39:50,061 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-22T03:39:50,061 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-22T03:39:50,062 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-22T03:39:50,066 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-22T03:39:50,071 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-22T03:39:50,072 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67772613, jitterRate=0.009890630841255188}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-22T03:39:50,075 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1732246790033Initializing all the Stores at 1732246790036 (+3 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732246790036Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732246790037 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732246790037Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732246790037Cleaning up temporary data from old regions at 1732246790061 (+24 ms)Region opened successfully at 1732246790075 (+14 ms) 2024-11-22T03:39:50,076 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-22T03:39:50,076 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-22T03:39:50,076 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-22T03:39:50,076 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-22T03:39:50,076 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-22T03:39:50,078 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-22T03:39:50,078 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732246790076Disabling compacts and flushes for region at 1732246790076Disabling writes for close at 1732246790076Writing region close event to WAL at 1732246790077 (+1 ms)Closed at 1732246790077 2024-11-22T03:39:50,083 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-22T03:39:50,083 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-22T03:39:50,091 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-22T03:39:50,100 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-22T03:39:50,103 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-22T03:39:50,259 DEBUG [c85114ed5096:42627 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-11-22T03:39:50,273 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=c85114ed5096,42545,1732246786482 2024-11-22T03:39:50,279 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as c85114ed5096,42545,1732246786482, state=OPENING 2024-11-22T03:39:50,283 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-22T03:39:50,284 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42545-0x1006583a7f00001, quorum=127.0.0.1:64888, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-22T03:39:50,284 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42627-0x1006583a7f00000, quorum=127.0.0.1:64888, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-22T03:39:50,285 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-22T03:39:50,285 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-22T03:39:50,287 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-22T03:39:50,289 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=c85114ed5096,42545,1732246786482}] 2024-11-22T03:39:50,467 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-22T03:39:50,470 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36693, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-22T03:39:50,484 INFO [RS_OPEN_META-regionserver/c85114ed5096:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-22T03:39:50,485 INFO [RS_OPEN_META-regionserver/c85114ed5096:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-22T03:39:50,485 INFO [RS_OPEN_META-regionserver/c85114ed5096:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-22T03:39:50,489 INFO [RS_OPEN_META-regionserver/c85114ed5096:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=c85114ed5096%2C42545%2C1732246786482.meta, suffix=.meta, logDir=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/WALs/c85114ed5096,42545,1732246786482, archiveDir=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/oldWALs, maxLogs=32 2024-11-22T03:39:50,512 DEBUG [RS_OPEN_META-regionserver/c85114ed5096:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/WALs/c85114ed5096,42545,1732246786482/c85114ed5096%2C42545%2C1732246786482.meta.1732246790492.meta, exclude list is [], retry=0 2024-11-22T03:39:50,516 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41449,DS-89103e69-741c-4fec-ac01-a162c6d86ae3,DISK] 2024-11-22T03:39:50,519 INFO [RS_OPEN_META-regionserver/c85114ed5096:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/WALs/c85114ed5096,42545,1732246786482/c85114ed5096%2C42545%2C1732246786482.meta.1732246790492.meta 2024-11-22T03:39:50,519 DEBUG [RS_OPEN_META-regionserver/c85114ed5096:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:38805:38805)] 2024-11-22T03:39:50,519 DEBUG [RS_OPEN_META-regionserver/c85114ed5096:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-22T03:39:50,521 DEBUG [RS_OPEN_META-regionserver/c85114ed5096:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-22T03:39:50,523 DEBUG [RS_OPEN_META-regionserver/c85114ed5096:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-22T03:39:50,528 INFO [RS_OPEN_META-regionserver/c85114ed5096:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-22T03:39:50,533 DEBUG [RS_OPEN_META-regionserver/c85114ed5096:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-22T03:39:50,533 DEBUG [RS_OPEN_META-regionserver/c85114ed5096:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-22T03:39:50,534 DEBUG [RS_OPEN_META-regionserver/c85114ed5096:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-22T03:39:50,534 DEBUG [RS_OPEN_META-regionserver/c85114ed5096:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-22T03:39:50,537 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-22T03:39:50,539 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-22T03:39:50,539 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:39:50,540 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-22T03:39:50,540 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-22T03:39:50,541 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-22T03:39:50,541 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:39:50,542 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-22T03:39:50,543 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-22T03:39:50,544 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-22T03:39:50,544 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:39:50,545 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-22T03:39:50,545 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-22T03:39:50,547 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-22T03:39:50,547 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:39:50,548 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-22T03:39:50,548 DEBUG [RS_OPEN_META-regionserver/c85114ed5096:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-22T03:39:50,549 DEBUG [RS_OPEN_META-regionserver/c85114ed5096:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/hbase/meta/1588230740 2024-11-22T03:39:50,552 DEBUG [RS_OPEN_META-regionserver/c85114ed5096:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/hbase/meta/1588230740 2024-11-22T03:39:50,554 DEBUG [RS_OPEN_META-regionserver/c85114ed5096:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-22T03:39:50,555 DEBUG [RS_OPEN_META-regionserver/c85114ed5096:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-22T03:39:50,555 DEBUG [RS_OPEN_META-regionserver/c85114ed5096:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-22T03:39:50,558 DEBUG [RS_OPEN_META-regionserver/c85114ed5096:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-22T03:39:50,560 INFO [RS_OPEN_META-regionserver/c85114ed5096:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67842853, jitterRate=0.010937288403511047}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-22T03:39:50,560 DEBUG [RS_OPEN_META-regionserver/c85114ed5096:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-22T03:39:50,561 DEBUG [RS_OPEN_META-regionserver/c85114ed5096:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1732246790534Writing region info on filesystem at 1732246790534Initializing all the Stores at 1732246790536 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732246790536Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732246790537 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732246790537Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732246790537Cleaning up temporary data from old regions at 1732246790555 (+18 ms)Running coprocessor post-open hooks at 1732246790560 (+5 ms)Region opened successfully at 1732246790561 (+1 ms) 2024-11-22T03:39:50,567 INFO [RS_OPEN_META-regionserver/c85114ed5096:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1732246790456 2024-11-22T03:39:50,577 DEBUG [RS_OPEN_META-regionserver/c85114ed5096:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-22T03:39:50,578 INFO [RS_OPEN_META-regionserver/c85114ed5096:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-22T03:39:50,581 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=c85114ed5096,42545,1732246786482 2024-11-22T03:39:50,583 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as c85114ed5096,42545,1732246786482, state=OPEN 2024-11-22T03:39:50,587 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42627-0x1006583a7f00000, quorum=127.0.0.1:64888, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-22T03:39:50,587 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42545-0x1006583a7f00001, quorum=127.0.0.1:64888, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-22T03:39:50,587 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-22T03:39:50,587 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-22T03:39:50,587 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=c85114ed5096,42545,1732246786482 2024-11-22T03:39:50,593 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-22T03:39:50,594 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=c85114ed5096,42545,1732246786482 in 300 msec 2024-11-22T03:39:50,600 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-22T03:39:50,600 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 505 msec 2024-11-22T03:39:50,602 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-22T03:39:50,602 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-22T03:39:50,620 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-22T03:39:50,621 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c85114ed5096,42545,1732246786482, seqNum=-1] 2024-11-22T03:39:50,639 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-22T03:39:50,642 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33025, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-22T03:39:50,663 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.5760 sec 2024-11-22T03:39:50,663 INFO [master/c85114ed5096:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1732246790663, completionTime=-1 2024-11-22T03:39:50,667 INFO [master/c85114ed5096:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-11-22T03:39:50,667 DEBUG [master/c85114ed5096:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-22T03:39:50,692 INFO [master/c85114ed5096:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1 2024-11-22T03:39:50,692 INFO [master/c85114ed5096:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732246850692 2024-11-22T03:39:50,692 INFO [master/c85114ed5096:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732246910692 2024-11-22T03:39:50,692 INFO [master/c85114ed5096:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 24 msec 2024-11-22T03:39:50,694 INFO [master/c85114ed5096:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=c85114ed5096,42627,1732246785796-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-22T03:39:50,694 INFO [master/c85114ed5096:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=c85114ed5096,42627,1732246785796-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-22T03:39:50,695 INFO [master/c85114ed5096:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=c85114ed5096,42627,1732246785796-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-22T03:39:50,696 INFO [master/c85114ed5096:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-c85114ed5096:42627, period=300000, unit=MILLISECONDS is enabled. 2024-11-22T03:39:50,696 INFO [master/c85114ed5096:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-22T03:39:50,697 INFO [master/c85114ed5096:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-22T03:39:50,703 DEBUG [master/c85114ed5096:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-22T03:39:50,725 INFO [master/c85114ed5096:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 4.085sec 2024-11-22T03:39:50,727 INFO [master/c85114ed5096:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-22T03:39:50,728 INFO [master/c85114ed5096:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-22T03:39:50,729 INFO [master/c85114ed5096:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-22T03:39:50,730 INFO [master/c85114ed5096:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-22T03:39:50,730 INFO [master/c85114ed5096:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-22T03:39:50,731 INFO [master/c85114ed5096:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=c85114ed5096,42627,1732246785796-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-22T03:39:50,732 INFO [master/c85114ed5096:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=c85114ed5096,42627,1732246785796-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-22T03:39:50,742 DEBUG [master/c85114ed5096:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-22T03:39:50,743 INFO [master/c85114ed5096:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-22T03:39:50,744 INFO [master/c85114ed5096:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=c85114ed5096,42627,1732246785796-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-22T03:39:50,802 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@45544f78, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T03:39:50,804 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-22T03:39:50,804 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-22T03:39:50,807 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c85114ed5096,42627,-1 for getting cluster id 2024-11-22T03:39:50,809 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-22T03:39:50,819 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '67035b7f-c649-44a8-9db7-17adff49cedb' 2024-11-22T03:39:50,822 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-22T03:39:50,822 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "67035b7f-c649-44a8-9db7-17adff49cedb" 2024-11-22T03:39:50,823 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@66836b1b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T03:39:50,823 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c85114ed5096,42627,-1] 2024-11-22T03:39:50,827 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-22T03:39:50,829 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T03:39:50,830 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48306, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-22T03:39:50,833 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2de67725, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T03:39:50,834 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-22T03:39:50,843 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c85114ed5096,42545,1732246786482, seqNum=-1] 2024-11-22T03:39:50,844 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-22T03:39:50,847 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49540, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-22T03:39:50,867 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=c85114ed5096,42627,1732246785796 2024-11-22T03:39:50,880 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testMixedAtomicity Thread=211, OpenFileDescriptor=441, MaxFileDescriptor=1048576, SystemLoadAverage=363, ProcessCount=11, AvailableMemoryMB=4033 2024-11-22T03:39:50,918 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-22T03:39:50,925 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncConnectionImpl(321): The fetched master address is c85114ed5096,42627,1732246785796 2024-11-22T03:39:50,927 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@30bfb0ba 2024-11-22T03:39:50,928 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-22T03:39:50,933 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48312, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-22T03:39:50,945 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42627 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-22T03:39:50,949 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42627 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-22T03:39:50,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42627 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-22T03:39:50,963 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-22T03:39:50,966 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42627 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 4 2024-11-22T03:39:50,966 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:39:50,969 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-22T03:39:50,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-22T03:39:50,986 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073741835_1011 (size=959) 2024-11-22T03:39:51,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-22T03:39:51,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-22T03:39:51,392 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => f970bd9f1e861e2b64a2fb87fb209ac4, NAME => 'TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a 2024-11-22T03:39:51,401 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073741836_1012 (size=53) 2024-11-22T03:39:51,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-22T03:39:51,803 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-22T03:39:51,803 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing f970bd9f1e861e2b64a2fb87fb209ac4, disabling compactions & flushes 2024-11-22T03:39:51,803 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4. 2024-11-22T03:39:51,803 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4. 2024-11-22T03:39:51,803 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4. after waiting 0 ms 2024-11-22T03:39:51,803 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4. 2024-11-22T03:39:51,803 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4. 2024-11-22T03:39:51,803 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for f970bd9f1e861e2b64a2fb87fb209ac4: Waiting for close lock at 1732246791803Disabling compacts and flushes for region at 1732246791803Disabling writes for close at 1732246791803Writing region close event to WAL at 1732246791803Closed at 1732246791803 2024-11-22T03:39:51,806 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-22T03:39:51,810 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732246791806"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732246791806"}]},"ts":"1732246791806"} 2024-11-22T03:39:51,815 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-22T03:39:51,817 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-22T03:39:51,820 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732246791817"}]},"ts":"1732246791817"} 2024-11-22T03:39:51,824 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-22T03:39:51,825 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f970bd9f1e861e2b64a2fb87fb209ac4, ASSIGN}] 2024-11-22T03:39:51,827 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f970bd9f1e861e2b64a2fb87fb209ac4, ASSIGN 2024-11-22T03:39:51,829 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=f970bd9f1e861e2b64a2fb87fb209ac4, ASSIGN; state=OFFLINE, location=c85114ed5096,42545,1732246786482; forceNewPlan=false, retain=false 2024-11-22T03:39:51,982 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=f970bd9f1e861e2b64a2fb87fb209ac4, regionState=OPENING, regionLocation=c85114ed5096,42545,1732246786482 2024-11-22T03:39:51,990 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=f970bd9f1e861e2b64a2fb87fb209ac4, ASSIGN because future has completed 2024-11-22T03:39:51,991 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482}] 2024-11-22T03:39:52,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-22T03:39:52,155 INFO [RS_OPEN_REGION-regionserver/c85114ed5096:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4. 2024-11-22T03:39:52,156 DEBUG [RS_OPEN_REGION-regionserver/c85114ed5096:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => f970bd9f1e861e2b64a2fb87fb209ac4, NAME => 'TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4.', STARTKEY => '', ENDKEY => ''} 2024-11-22T03:39:52,156 DEBUG [RS_OPEN_REGION-regionserver/c85114ed5096:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees f970bd9f1e861e2b64a2fb87fb209ac4 2024-11-22T03:39:52,157 DEBUG [RS_OPEN_REGION-regionserver/c85114ed5096:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-22T03:39:52,157 DEBUG [RS_OPEN_REGION-regionserver/c85114ed5096:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for f970bd9f1e861e2b64a2fb87fb209ac4 2024-11-22T03:39:52,157 DEBUG [RS_OPEN_REGION-regionserver/c85114ed5096:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for f970bd9f1e861e2b64a2fb87fb209ac4 2024-11-22T03:39:52,161 INFO [StoreOpener-f970bd9f1e861e2b64a2fb87fb209ac4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region f970bd9f1e861e2b64a2fb87fb209ac4 2024-11-22T03:39:52,163 INFO [StoreOpener-f970bd9f1e861e2b64a2fb87fb209ac4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f970bd9f1e861e2b64a2fb87fb209ac4 columnFamilyName A 2024-11-22T03:39:52,163 DEBUG [StoreOpener-f970bd9f1e861e2b64a2fb87fb209ac4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:39:52,164 INFO [StoreOpener-f970bd9f1e861e2b64a2fb87fb209ac4-1 {}] regionserver.HStore(327): Store=f970bd9f1e861e2b64a2fb87fb209ac4/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-22T03:39:52,164 INFO [StoreOpener-f970bd9f1e861e2b64a2fb87fb209ac4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region f970bd9f1e861e2b64a2fb87fb209ac4 2024-11-22T03:39:52,166 INFO [StoreOpener-f970bd9f1e861e2b64a2fb87fb209ac4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f970bd9f1e861e2b64a2fb87fb209ac4 columnFamilyName B 2024-11-22T03:39:52,166 DEBUG [StoreOpener-f970bd9f1e861e2b64a2fb87fb209ac4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:39:52,167 INFO [StoreOpener-f970bd9f1e861e2b64a2fb87fb209ac4-1 {}] regionserver.HStore(327): Store=f970bd9f1e861e2b64a2fb87fb209ac4/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-22T03:39:52,167 INFO [StoreOpener-f970bd9f1e861e2b64a2fb87fb209ac4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region f970bd9f1e861e2b64a2fb87fb209ac4 2024-11-22T03:39:52,169 INFO [StoreOpener-f970bd9f1e861e2b64a2fb87fb209ac4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f970bd9f1e861e2b64a2fb87fb209ac4 columnFamilyName C 2024-11-22T03:39:52,170 DEBUG [StoreOpener-f970bd9f1e861e2b64a2fb87fb209ac4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:39:52,170 INFO [StoreOpener-f970bd9f1e861e2b64a2fb87fb209ac4-1 {}] regionserver.HStore(327): Store=f970bd9f1e861e2b64a2fb87fb209ac4/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-22T03:39:52,171 DEBUG [RS_OPEN_REGION-regionserver/c85114ed5096:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for f970bd9f1e861e2b64a2fb87fb209ac4 2024-11-22T03:39:52,172 DEBUG [RS_OPEN_REGION-regionserver/c85114ed5096:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4 2024-11-22T03:39:52,173 DEBUG [RS_OPEN_REGION-regionserver/c85114ed5096:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4 2024-11-22T03:39:52,175 DEBUG [RS_OPEN_REGION-regionserver/c85114ed5096:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for f970bd9f1e861e2b64a2fb87fb209ac4 2024-11-22T03:39:52,175 DEBUG [RS_OPEN_REGION-regionserver/c85114ed5096:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for f970bd9f1e861e2b64a2fb87fb209ac4 2024-11-22T03:39:52,176 DEBUG [RS_OPEN_REGION-regionserver/c85114ed5096:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-22T03:39:52,179 DEBUG [RS_OPEN_REGION-regionserver/c85114ed5096:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for f970bd9f1e861e2b64a2fb87fb209ac4 2024-11-22T03:39:52,183 DEBUG [RS_OPEN_REGION-regionserver/c85114ed5096:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-22T03:39:52,184 INFO [RS_OPEN_REGION-regionserver/c85114ed5096:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened f970bd9f1e861e2b64a2fb87fb209ac4; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61529959, jitterRate=-0.08313216269016266}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-22T03:39:52,184 DEBUG [RS_OPEN_REGION-regionserver/c85114ed5096:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for f970bd9f1e861e2b64a2fb87fb209ac4 2024-11-22T03:39:52,185 DEBUG [RS_OPEN_REGION-regionserver/c85114ed5096:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for f970bd9f1e861e2b64a2fb87fb209ac4: Running coprocessor pre-open hook at 1732246792157Writing region info on filesystem at 1732246792157Initializing all the Stores at 1732246792160 (+3 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732246792160Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732246792160Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732246792161 (+1 ms)Cleaning up temporary data from old regions at 1732246792175 (+14 ms)Running coprocessor post-open hooks at 1732246792184 (+9 ms)Region opened successfully at 1732246792185 (+1 ms) 2024-11-22T03:39:52,188 INFO [RS_OPEN_REGION-regionserver/c85114ed5096:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., pid=6, masterSystemTime=1732246792146 2024-11-22T03:39:52,192 DEBUG [RS_OPEN_REGION-regionserver/c85114ed5096:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4. 2024-11-22T03:39:52,192 INFO [RS_OPEN_REGION-regionserver/c85114ed5096:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4. 2024-11-22T03:39:52,193 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=f970bd9f1e861e2b64a2fb87fb209ac4, regionState=OPEN, openSeqNum=2, regionLocation=c85114ed5096,42545,1732246786482 2024-11-22T03:39:52,198 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 because future has completed 2024-11-22T03:39:52,205 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-22T03:39:52,205 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 in 209 msec 2024-11-22T03:39:52,211 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-22T03:39:52,211 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f970bd9f1e861e2b64a2fb87fb209ac4, ASSIGN in 380 msec 2024-11-22T03:39:52,214 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-22T03:39:52,214 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732246792214"}]},"ts":"1732246792214"} 2024-11-22T03:39:52,219 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-22T03:39:52,221 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-22T03:39:52,227 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.2690 sec 2024-11-22T03:39:53,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-22T03:39:53,120 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-22T03:39:53,126 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@775e68a3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T03:39:53,127 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c85114ed5096,42627,-1 for getting cluster id 2024-11-22T03:39:53,127 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-22T03:39:53,129 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '67035b7f-c649-44a8-9db7-17adff49cedb' 2024-11-22T03:39:53,130 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-22T03:39:53,130 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "67035b7f-c649-44a8-9db7-17adff49cedb" 2024-11-22T03:39:53,130 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3f9668b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T03:39:53,131 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c85114ed5096,42627,-1] 2024-11-22T03:39:53,131 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-22T03:39:53,132 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T03:39:53,133 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48338, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-22T03:39:53,135 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5a5677a3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T03:39:53,136 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-22T03:39:53,138 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c85114ed5096,42545,1732246786482, seqNum=-1] 2024-11-22T03:39:53,139 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-22T03:39:53,142 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49556, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-22T03:39:53,145 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-22T03:39:53,146 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is c85114ed5096,42627,1732246785796 2024-11-22T03:39:53,146 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@2c4ff25b 2024-11-22T03:39:53,149 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-22T03:39:53,151 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48348, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-22T03:39:53,156 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@392f723b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T03:39:53,156 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c85114ed5096,42627,-1 for getting cluster id 2024-11-22T03:39:53,156 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-22T03:39:53,158 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '67035b7f-c649-44a8-9db7-17adff49cedb' 2024-11-22T03:39:53,159 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-22T03:39:53,159 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "67035b7f-c649-44a8-9db7-17adff49cedb" 2024-11-22T03:39:53,159 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1eb99d17, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T03:39:53,159 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c85114ed5096,42627,-1] 2024-11-22T03:39:53,160 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-22T03:39:53,160 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T03:39:53,161 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48376, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-22T03:39:53,163 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7aafb42a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T03:39:53,164 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6f0e98fe, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T03:39:53,164 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c85114ed5096,42627,-1 for getting cluster id 2024-11-22T03:39:53,164 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-22T03:39:53,165 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '67035b7f-c649-44a8-9db7-17adff49cedb' 2024-11-22T03:39:53,166 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-22T03:39:53,166 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "67035b7f-c649-44a8-9db7-17adff49cedb" 2024-11-22T03:39:53,166 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@654d6699, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T03:39:53,166 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c85114ed5096,42627,-1] 2024-11-22T03:39:53,166 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-22T03:39:53,168 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T03:39:53,169 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48386, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-22T03:39:53,170 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@688e5843, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T03:39:53,171 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@69066494, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T03:39:53,171 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c85114ed5096,42627,-1 for getting cluster id 2024-11-22T03:39:53,172 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-22T03:39:53,173 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '67035b7f-c649-44a8-9db7-17adff49cedb' 2024-11-22T03:39:53,174 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-22T03:39:53,174 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "67035b7f-c649-44a8-9db7-17adff49cedb" 2024-11-22T03:39:53,175 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6c373c50, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T03:39:53,175 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c85114ed5096,42627,-1] 2024-11-22T03:39:53,175 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-22T03:39:53,177 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T03:39:53,178 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48404, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-22T03:39:53,180 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4834e559, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T03:39:53,180 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3f00f692, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T03:39:53,180 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c85114ed5096,42627,-1 for getting cluster id 2024-11-22T03:39:53,181 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-22T03:39:53,182 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '67035b7f-c649-44a8-9db7-17adff49cedb' 2024-11-22T03:39:53,183 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-22T03:39:53,183 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "67035b7f-c649-44a8-9db7-17adff49cedb" 2024-11-22T03:39:53,183 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@44447930, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T03:39:53,183 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c85114ed5096,42627,-1] 2024-11-22T03:39:53,184 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-22T03:39:53,184 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T03:39:53,186 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48426, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-22T03:39:53,187 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@537ae38c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T03:39:53,188 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@388c382, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T03:39:53,188 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c85114ed5096,42627,-1 for getting cluster id 2024-11-22T03:39:53,188 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-22T03:39:53,190 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '67035b7f-c649-44a8-9db7-17adff49cedb' 2024-11-22T03:39:53,190 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-22T03:39:53,190 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "67035b7f-c649-44a8-9db7-17adff49cedb" 2024-11-22T03:39:53,190 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6ee3c9e1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T03:39:53,190 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c85114ed5096,42627,-1] 2024-11-22T03:39:53,191 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-22T03:39:53,191 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T03:39:53,192 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48440, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-22T03:39:53,193 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7b594d15, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T03:39:53,195 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7ff89310, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T03:39:53,195 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c85114ed5096,42627,-1 for getting cluster id 2024-11-22T03:39:53,195 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-22T03:39:53,196 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '67035b7f-c649-44a8-9db7-17adff49cedb' 2024-11-22T03:39:53,197 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-22T03:39:53,197 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "67035b7f-c649-44a8-9db7-17adff49cedb" 2024-11-22T03:39:53,197 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@59ebb6cd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T03:39:53,197 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c85114ed5096,42627,-1] 2024-11-22T03:39:53,198 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-22T03:39:53,198 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T03:39:53,199 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48464, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-22T03:39:53,200 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7d52c48, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T03:39:53,204 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@761fbe54, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T03:39:53,204 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c85114ed5096,42627,-1 for getting cluster id 2024-11-22T03:39:53,205 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-22T03:39:53,206 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '67035b7f-c649-44a8-9db7-17adff49cedb' 2024-11-22T03:39:53,206 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-22T03:39:53,207 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "67035b7f-c649-44a8-9db7-17adff49cedb" 2024-11-22T03:39:53,207 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@cc2de1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T03:39:53,207 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c85114ed5096,42627,-1] 2024-11-22T03:39:53,207 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-22T03:39:53,208 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T03:39:53,209 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48484, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-22T03:39:53,210 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@47c48967, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T03:39:53,211 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2019371a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T03:39:53,211 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c85114ed5096,42627,-1 for getting cluster id 2024-11-22T03:39:53,211 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-22T03:39:53,212 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '67035b7f-c649-44a8-9db7-17adff49cedb' 2024-11-22T03:39:53,212 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-22T03:39:53,213 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "67035b7f-c649-44a8-9db7-17adff49cedb" 2024-11-22T03:39:53,213 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@775e595a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T03:39:53,213 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c85114ed5096,42627,-1] 2024-11-22T03:39:53,213 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-22T03:39:53,213 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T03:39:53,214 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48500, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-22T03:39:53,215 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4852f5b0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T03:39:53,216 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@36df70c8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T03:39:53,216 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c85114ed5096,42627,-1 for getting cluster id 2024-11-22T03:39:53,216 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-22T03:39:53,217 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '67035b7f-c649-44a8-9db7-17adff49cedb' 2024-11-22T03:39:53,217 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-22T03:39:53,218 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "67035b7f-c649-44a8-9db7-17adff49cedb" 2024-11-22T03:39:53,218 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@331f9727, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T03:39:53,218 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c85114ed5096,42627,-1] 2024-11-22T03:39:53,218 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-22T03:39:53,218 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T03:39:53,219 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48514, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-22T03:39:53,220 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2fcf1edd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T03:39:53,224 DEBUG [Thread-147 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-22T03:39:53,224 DEBUG [Thread-156 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-22T03:39:53,224 DEBUG [Thread-143 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-22T03:39:53,224 DEBUG [Thread-158 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-22T03:39:53,224 DEBUG [Thread-160 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-22T03:39:53,225 DEBUG [Thread-149 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-22T03:39:53,226 DEBUG [Thread-154 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-22T03:39:53,226 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c85114ed5096,42545,1732246786482, seqNum=-1] 2024-11-22T03:39:53,226 DEBUG [Thread-151 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-22T03:39:53,226 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c85114ed5096,42545,1732246786482, seqNum=-1] 2024-11-22T03:39:53,226 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-22T03:39:53,226 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c85114ed5096,42545,1732246786482, seqNum=-1] 2024-11-22T03:39:53,226 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-22T03:39:53,226 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-22T03:39:53,227 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42627 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-22T03:39:53,227 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c85114ed5096,42545,1732246786482, seqNum=-1] 2024-11-22T03:39:53,227 DEBUG [Thread-145 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-22T03:39:53,227 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-22T03:39:53,228 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c85114ed5096,42545,1732246786482, seqNum=-1] 2024-11-22T03:39:53,228 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49566, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-22T03:39:53,228 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-22T03:39:53,228 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c85114ed5096,42545,1732246786482, seqNum=-1] 2024-11-22T03:39:53,228 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-22T03:39:53,229 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49568, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-22T03:39:53,229 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49572, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-22T03:39:53,229 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c85114ed5096,42545,1732246786482, seqNum=-1] 2024-11-22T03:39:53,230 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-22T03:39:53,230 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49590, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-22T03:39:53,230 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c85114ed5096,42545,1732246786482, seqNum=-1] 2024-11-22T03:39:53,230 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-22T03:39:53,230 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c85114ed5096,42545,1732246786482, seqNum=-1] 2024-11-22T03:39:53,231 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-22T03:39:53,231 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49582, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-22T03:39:53,231 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49602, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-22T03:39:53,232 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49606, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-22T03:39:53,232 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49612, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-22T03:39:53,232 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49626, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-22T03:39:53,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42627 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees 2024-11-22T03:39:53,237 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2] 2024-11-22T03:39:53,238 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-22T03:39:53,238 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2] 2024-11-22T03:39:53,238 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2] 2024-11-22T03:39:53,238 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2] 2024-11-22T03:39:53,240 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-22T03:39:53,242 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-22T03:39:53,243 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2] 2024-11-22T03:39:53,243 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2] 2024-11-22T03:39:53,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-22T03:39:53,246 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2] 2024-11-22T03:39:53,247 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2] 2024-11-22T03:39:53,251 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2] 2024-11-22T03:39:53,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(8855): Flush requested on f970bd9f1e861e2b64a2fb87fb209ac4 2024-11-22T03:39:53,322 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-22T03:39:53,327 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f970bd9f1e861e2b64a2fb87fb209ac4 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-22T03:39:53,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-22T03:39:53,428 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42545 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-22T03:39:53,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4. 2024-11-22T03:39:53,430 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4. as already flushing 2024-11-22T03:39:53,430 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4. 2024-11-22T03:39:53,431 ERROR [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => f970bd9f1e861e2b64a2fb87fb209ac4, NAME => 'TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:39:53,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => f970bd9f1e861e2b64a2fb87fb209ac4, NAME => 'TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:39:53,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42627 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f970bd9f1e861e2b64a2fb87fb209ac4, NAME => 'TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f970bd9f1e861e2b64a2fb87fb209ac4, NAME => 'TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:39:53,516 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/.tmp/A/7707745419cb47b79b1297ab3f9881bb is 50, key is test_row_0/A:col10/1732246793312/Put/seqid=0 2024-11-22T03:39:53,534 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073741837_1013 (size=12001) 2024-11-22T03:39:53,554 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:39:53,558 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:39:53,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-22T03:39:53,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49582 deadline: 1732246853548, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:39:53,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49590 deadline: 1732246853525, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:39:53,564 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:39:53,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49606 deadline: 1732246853549, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:39:53,565 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:39:53,566 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:39:53,566 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:39:53,566 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:39:53,567 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:39:53,567 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:39:53,567 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:39:53,567 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:39:53,568 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:39:53,572 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:39:53,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49566 deadline: 1732246853555, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:39:53,573 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:39:53,573 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:39:53,573 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:39:53,576 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:39:53,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49612 deadline: 1732246853559, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:39:53,578 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:39:53,578 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:39:53,578 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:39:53,608 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42545 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-22T03:39:53,609 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4. 2024-11-22T03:39:53,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4. as already flushing 2024-11-22T03:39:53,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4. 2024-11-22T03:39:53,613 ERROR [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => f970bd9f1e861e2b64a2fb87fb209ac4, NAME => 'TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:39:53,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => f970bd9f1e861e2b64a2fb87fb209ac4, NAME => 'TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:39:53,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42627 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f970bd9f1e861e2b64a2fb87fb209ac4, NAME => 'TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f970bd9f1e861e2b64a2fb87fb209ac4, NAME => 'TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:39:53,671 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:39:53,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49590 deadline: 1732246853669, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:39:53,673 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:39:53,673 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:39:53,673 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:39:53,683 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:39:53,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49606 deadline: 1732246853681, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:39:53,684 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:39:53,684 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:39:53,685 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:39:53,685 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:39:53,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49566 deadline: 1732246853682, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:39:53,682 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:39:53,685 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:39:53,685 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:39:53,686 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:39:53,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49582 deadline: 1732246853680, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:39:53,686 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:39:53,686 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:39:53,686 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:39:53,691 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:39:53,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49612 deadline: 1732246853690, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:39:53,692 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:39:53,692 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:39:53,693 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:39:53,779 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42545 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-22T03:39:53,779 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4. 2024-11-22T03:39:53,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4. as already flushing 2024-11-22T03:39:53,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4. 2024-11-22T03:39:53,780 ERROR [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => f970bd9f1e861e2b64a2fb87fb209ac4, NAME => 'TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:39:53,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => f970bd9f1e861e2b64a2fb87fb209ac4, NAME => 'TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:39:53,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42627 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f970bd9f1e861e2b64a2fb87fb209ac4, NAME => 'TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f970bd9f1e861e2b64a2fb87fb209ac4, NAME => 'TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:39:53,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-22T03:39:53,882 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:39:53,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49590 deadline: 1732246853880, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:39:53,883 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:39:53,883 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:39:53,883 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:39:53,892 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:39:53,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49566 deadline: 1732246853891, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:39:53,893 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:39:53,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49606 deadline: 1732246853890, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:39:53,894 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:39:53,894 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:39:53,894 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:39:53,895 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:39:53,895 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:39:53,895 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:39:53,902 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:39:53,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49582 deadline: 1732246853900, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:39:53,903 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:39:53,903 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:39:53,903 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:39:53,903 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:39:53,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49612 deadline: 1732246853901, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:39:53,904 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:39:53,905 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:39:53,905 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:39:53,936 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42545 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-22T03:39:53,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4. 2024-11-22T03:39:53,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4. as already flushing 2024-11-22T03:39:53,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4. 2024-11-22T03:39:53,936 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/.tmp/A/7707745419cb47b79b1297ab3f9881bb 2024-11-22T03:39:53,936 ERROR [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => f970bd9f1e861e2b64a2fb87fb209ac4, NAME => 'TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:39:53,937 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => f970bd9f1e861e2b64a2fb87fb209ac4, NAME => 'TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:39:53,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42627 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f970bd9f1e861e2b64a2fb87fb209ac4, NAME => 'TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f970bd9f1e861e2b64a2fb87fb209ac4, NAME => 'TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:39:54,074 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/.tmp/B/920b0be291b247aa8d976946b061290c is 50, key is test_row_0/B:col10/1732246793312/Put/seqid=0 2024-11-22T03:39:54,091 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073741838_1014 (size=12001) 2024-11-22T03:39:54,093 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42545 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-22T03:39:54,093 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4. 2024-11-22T03:39:54,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4. as already flushing 2024-11-22T03:39:54,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4. 2024-11-22T03:39:54,094 ERROR [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => f970bd9f1e861e2b64a2fb87fb209ac4, NAME => 'TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:39:54,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => f970bd9f1e861e2b64a2fb87fb209ac4, NAME => 'TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:39:54,096 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/.tmp/B/920b0be291b247aa8d976946b061290c 2024-11-22T03:39:54,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42627 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f970bd9f1e861e2b64a2fb87fb209ac4, NAME => 'TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f970bd9f1e861e2b64a2fb87fb209ac4, NAME => 'TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:39:54,150 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/.tmp/C/8a9d616772a64f4294051324e940a5ca is 50, key is test_row_0/C:col10/1732246793312/Put/seqid=0 2024-11-22T03:39:54,177 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073741839_1015 (size=12001) 2024-11-22T03:39:54,180 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/.tmp/C/8a9d616772a64f4294051324e940a5ca 2024-11-22T03:39:54,195 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:39:54,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49590 deadline: 1732246854190, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:39:54,197 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:39:54,197 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:39:54,197 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:39:54,200 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:39:54,201 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:39:54,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49566 deadline: 1732246854199, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:39:54,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49606 deadline: 1732246854200, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:39:54,201 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/.tmp/A/7707745419cb47b79b1297ab3f9881bb as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/A/7707745419cb47b79b1297ab3f9881bb 2024-11-22T03:39:54,202 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:39:54,202 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:39:54,202 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:39:54,202 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:39:54,202 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:39:54,202 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:39:54,211 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:39:54,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49582 deadline: 1732246854209, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:39:54,212 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:39:54,212 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:39:54,212 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:39:54,212 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:39:54,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49612 deadline: 1732246854209, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:39:54,214 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:39:54,214 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:39:54,214 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:39:54,220 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/A/7707745419cb47b79b1297ab3f9881bb, entries=150, sequenceid=14, filesize=11.7 K 2024-11-22T03:39:54,226 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreScanner(1005): StoreScanner already closing. There is no need to updateReaders 2024-11-22T03:39:54,229 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/.tmp/B/920b0be291b247aa8d976946b061290c as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/B/920b0be291b247aa8d976946b061290c 2024-11-22T03:39:54,247 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/B/920b0be291b247aa8d976946b061290c, entries=150, sequenceid=14, filesize=11.7 K 2024-11-22T03:39:54,250 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/.tmp/C/8a9d616772a64f4294051324e940a5ca as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/C/8a9d616772a64f4294051324e940a5ca 2024-11-22T03:39:54,252 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42545 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-22T03:39:54,252 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4. 2024-11-22T03:39:54,252 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4. as already flushing 2024-11-22T03:39:54,252 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4. 2024-11-22T03:39:54,252 ERROR [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => f970bd9f1e861e2b64a2fb87fb209ac4, NAME => 'TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:39:54,253 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => f970bd9f1e861e2b64a2fb87fb209ac4, NAME => 'TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:39:54,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42627 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f970bd9f1e861e2b64a2fb87fb209ac4, NAME => 'TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f970bd9f1e861e2b64a2fb87fb209ac4, NAME => 'TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:39:54,272 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/C/8a9d616772a64f4294051324e940a5ca, entries=150, sequenceid=14, filesize=11.7 K 2024-11-22T03:39:54,282 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for f970bd9f1e861e2b64a2fb87fb209ac4 in 953ms, sequenceid=14, compaction requested=false 2024-11-22T03:39:54,283 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-22T03:39:54,286 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f970bd9f1e861e2b64a2fb87fb209ac4: 2024-11-22T03:39:54,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-22T03:39:54,408 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42545 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-22T03:39:54,409 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4. 2024-11-22T03:39:54,409 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2902): Flushing f970bd9f1e861e2b64a2fb87fb209ac4 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-22T03:39:54,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/.tmp/A/cd4945c96adf4832a8d7671f9f41d297 is 50, key is test_row_0/A:col10/1732246793545/Put/seqid=0 2024-11-22T03:39:54,470 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073741840_1016 (size=12001) 2024-11-22T03:39:54,471 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/.tmp/A/cd4945c96adf4832a8d7671f9f41d297 2024-11-22T03:39:54,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/.tmp/B/7acdd5c62ba64939a5cf0a87243c40ce is 50, key is test_row_0/B:col10/1732246793545/Put/seqid=0 2024-11-22T03:39:54,559 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073741841_1017 (size=12001) 2024-11-22T03:39:54,722 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4. as already flushing 2024-11-22T03:39:54,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(8855): Flush requested on f970bd9f1e861e2b64a2fb87fb209ac4 2024-11-22T03:39:54,769 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:39:54,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49590 deadline: 1732246854749, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:39:54,770 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:39:54,771 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:39:54,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49582 deadline: 1732246854758, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:39:54,771 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:39:54,771 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:39:54,771 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:39:54,771 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:39:54,772 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:39:54,771 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:39:54,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49566 deadline: 1732246854754, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:39:54,772 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:39:54,772 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:39:54,773 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:39:54,773 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:39:54,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49612 deadline: 1732246854770, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:39:54,774 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:39:54,774 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:39:54,774 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:39:54,774 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:39:54,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49606 deadline: 1732246854770, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:39:54,775 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:39:54,775 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:39:54,775 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:39:54,882 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:39:54,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49582 deadline: 1732246854881, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:39:54,883 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:39:54,884 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:39:54,884 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:39:54,884 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:39:54,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49566 deadline: 1732246854881, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:39:54,885 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:39:54,885 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:39:54,885 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:39:54,885 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:39:54,885 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:39:54,884 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:39:54,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49606 deadline: 1732246854882, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:39:54,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49590 deadline: 1732246854882, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:39:54,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49612 deadline: 1732246854882, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:39:54,886 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:39:54,886 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:39:54,886 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:39:54,887 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:39:54,887 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:39:54,887 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:39:54,887 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:39:54,887 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:39:54,887 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:39:54,961 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/.tmp/B/7acdd5c62ba64939a5cf0a87243c40ce 2024-11-22T03:39:54,984 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/.tmp/C/eda6acb3909c4de0a95ab22dd66ca079 is 50, key is test_row_0/C:col10/1732246793545/Put/seqid=0 2024-11-22T03:39:55,009 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073741842_1018 (size=12001) 2024-11-22T03:39:55,012 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/.tmp/C/eda6acb3909c4de0a95ab22dd66ca079 2024-11-22T03:39:55,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/.tmp/A/cd4945c96adf4832a8d7671f9f41d297 as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/A/cd4945c96adf4832a8d7671f9f41d297 2024-11-22T03:39:55,082 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/A/cd4945c96adf4832a8d7671f9f41d297, entries=150, sequenceid=37, filesize=11.7 K 2024-11-22T03:39:55,090 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/.tmp/B/7acdd5c62ba64939a5cf0a87243c40ce as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/B/7acdd5c62ba64939a5cf0a87243c40ce 2024-11-22T03:39:55,096 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:39:55,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49582 deadline: 1732246855090, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:39:55,097 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:39:55,097 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:39:55,097 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:39:55,097 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:39:55,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49590 deadline: 1732246855090, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:39:55,098 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:39:55,098 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:39:55,099 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:39:55,099 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:39:55,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49566 deadline: 1732246855091, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:39:55,100 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:39:55,100 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:39:55,100 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:39:55,102 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:39:55,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49612 deadline: 1732246855100, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:39:55,103 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:39:55,103 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:39:55,103 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:39:55,103 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:39:55,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49606 deadline: 1732246855100, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:39:55,104 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:39:55,104 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:39:55,104 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:39:55,123 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/B/7acdd5c62ba64939a5cf0a87243c40ce, entries=150, sequenceid=37, filesize=11.7 K 2024-11-22T03:39:55,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/.tmp/C/eda6acb3909c4de0a95ab22dd66ca079 as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/C/eda6acb3909c4de0a95ab22dd66ca079 2024-11-22T03:39:55,155 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/C/eda6acb3909c4de0a95ab22dd66ca079, entries=150, sequenceid=37, filesize=11.7 K 2024-11-22T03:39:55,157 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=73.80 KB/75570 for f970bd9f1e861e2b64a2fb87fb209ac4 in 748ms, sequenceid=37, compaction requested=false 2024-11-22T03:39:55,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2603): Flush status journal for f970bd9f1e861e2b64a2fb87fb209ac4: 2024-11-22T03:39:55,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4. 2024-11-22T03:39:55,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=8 2024-11-22T03:39:55,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42627 {}] master.HMaster(4169): Remote procedure done, pid=8 2024-11-22T03:39:55,171 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-22T03:39:55,171 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9190 sec 2024-11-22T03:39:55,181 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees in 1.9430 sec 2024-11-22T03:39:55,368 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-22T03:39:55,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-22T03:39:55,400 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-22T03:39:55,400 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42627 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-22T03:39:55,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42627 {}] procedure2.ProcedureExecutor(1139): Stored pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees 2024-11-22T03:39:55,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-22T03:39:55,404 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-22T03:39:55,407 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-22T03:39:55,407 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=10, ppid=9, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-22T03:39:55,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(8855): Flush requested on f970bd9f1e861e2b64a2fb87fb209ac4 2024-11-22T03:39:55,418 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-22T03:39:55,418 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f970bd9f1e861e2b64a2fb87fb209ac4 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-22T03:39:55,432 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/.tmp/A/cfb481a93dfa49358f7f828d17d57846 is 50, key is test_row_0/A:col10/1732246795413/Put/seqid=0 2024-11-22T03:39:55,468 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073741843_1019 (size=12001) 2024-11-22T03:39:55,470 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=54 (bloomFilter=true), to=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/.tmp/A/cfb481a93dfa49358f7f828d17d57846 2024-11-22T03:39:55,475 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:39:55,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49590 deadline: 1732246855463, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:39:55,477 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:39:55,477 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:39:55,477 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:39:55,480 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:39:55,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49606 deadline: 1732246855472, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:39:55,481 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:39:55,481 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:39:55,481 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:39:55,485 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:39:55,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49582 deadline: 1732246855472, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:39:55,486 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:39:55,487 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:39:55,487 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:39:55,487 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:39:55,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49566 deadline: 1732246855476, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:39:55,488 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:39:55,488 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:39:55,488 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:39:55,488 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:39:55,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49612 deadline: 1732246855476, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:39:55,489 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:39:55,489 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:39:55,489 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:39:55,492 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/.tmp/B/f4065c9b2c26424fa79e3683eee8f910 is 50, key is test_row_0/B:col10/1732246795413/Put/seqid=0 2024-11-22T03:39:55,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-22T03:39:55,515 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073741844_1020 (size=12001) 2024-11-22T03:39:55,519 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=54 (bloomFilter=true), to=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/.tmp/B/f4065c9b2c26424fa79e3683eee8f910 2024-11-22T03:39:55,547 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/.tmp/C/e232665be81a42d3b8e5ab310626de53 is 50, key is test_row_0/C:col10/1732246795413/Put/seqid=0 2024-11-22T03:39:55,565 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42545 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-22T03:39:55,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4. 2024-11-22T03:39:55,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4. as already flushing 2024-11-22T03:39:55,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4. 2024-11-22T03:39:55,566 ERROR [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => f970bd9f1e861e2b64a2fb87fb209ac4, NAME => 'TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:39:55,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => f970bd9f1e861e2b64a2fb87fb209ac4, NAME => 'TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:39:55,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42627 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f970bd9f1e861e2b64a2fb87fb209ac4, NAME => 'TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f970bd9f1e861e2b64a2fb87fb209ac4, NAME => 'TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:39:55,575 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073741845_1021 (size=12001) 2024-11-22T03:39:55,577 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=54 (bloomFilter=true), to=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/.tmp/C/e232665be81a42d3b8e5ab310626de53 2024-11-22T03:39:55,591 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/.tmp/A/cfb481a93dfa49358f7f828d17d57846 as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/A/cfb481a93dfa49358f7f828d17d57846 2024-11-22T03:39:55,591 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:39:55,591 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:39:55,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49590 deadline: 1732246855590, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:39:55,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49606 deadline: 1732246855590, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:39:55,592 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:39:55,592 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:39:55,592 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:39:55,592 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:39:55,593 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:39:55,593 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:39:55,592 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:39:55,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49582 deadline: 1732246855591, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:39:55,594 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:39:55,594 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:39:55,594 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:39:55,603 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:39:55,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49566 deadline: 1732246855599, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:39:55,604 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:39:55,604 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:39:55,604 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:39:55,605 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:39:55,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49612 deadline: 1732246855599, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:39:55,606 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:39:55,606 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:39:55,606 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:39:55,609 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/A/cfb481a93dfa49358f7f828d17d57846, entries=150, sequenceid=54, filesize=11.7 K 2024-11-22T03:39:55,612 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/.tmp/B/f4065c9b2c26424fa79e3683eee8f910 as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/B/f4065c9b2c26424fa79e3683eee8f910 2024-11-22T03:39:55,627 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/B/f4065c9b2c26424fa79e3683eee8f910, entries=150, sequenceid=54, filesize=11.7 K 2024-11-22T03:39:55,638 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/.tmp/C/e232665be81a42d3b8e5ab310626de53 as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/C/e232665be81a42d3b8e5ab310626de53 2024-11-22T03:39:55,655 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/C/e232665be81a42d3b8e5ab310626de53, entries=150, sequenceid=54, filesize=11.7 K 2024-11-22T03:39:55,662 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~246.80 KB/252720, currentSize=114.05 KB/116790 for f970bd9f1e861e2b64a2fb87fb209ac4 in 244ms, sequenceid=54, compaction requested=true 2024-11-22T03:39:55,662 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f970bd9f1e861e2b64a2fb87fb209ac4: 2024-11-22T03:39:55,666 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f970bd9f1e861e2b64a2fb87fb209ac4:A, priority=-2147483648, current under compaction store size is 1 2024-11-22T03:39:55,669 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T03:39:55,669 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f970bd9f1e861e2b64a2fb87fb209ac4:B, priority=-2147483648, current under compaction store size is 2 2024-11-22T03:39:55,669 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T03:39:55,669 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f970bd9f1e861e2b64a2fb87fb209ac4:C, priority=-2147483648, current under compaction store size is 3 2024-11-22T03:39:55,669 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T03:39:55,670 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T03:39:55,672 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T03:39:55,676 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T03:39:55,676 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T03:39:55,678 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HStore(1541): f970bd9f1e861e2b64a2fb87fb209ac4/B is initiating minor compaction (all files) 2024-11-22T03:39:55,678 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.HStore(1541): f970bd9f1e861e2b64a2fb87fb209ac4/A is initiating minor compaction (all files) 2024-11-22T03:39:55,678 INFO [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f970bd9f1e861e2b64a2fb87fb209ac4/A in TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4. 2024-11-22T03:39:55,678 INFO [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f970bd9f1e861e2b64a2fb87fb209ac4/B in TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4. 2024-11-22T03:39:55,679 INFO [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/A/7707745419cb47b79b1297ab3f9881bb, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/A/cd4945c96adf4832a8d7671f9f41d297, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/A/cfb481a93dfa49358f7f828d17d57846] into tmpdir=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/.tmp, totalSize=35.2 K 2024-11-22T03:39:55,681 INFO [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/B/920b0be291b247aa8d976946b061290c, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/B/7acdd5c62ba64939a5cf0a87243c40ce, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/B/f4065c9b2c26424fa79e3683eee8f910] into tmpdir=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/.tmp, totalSize=35.2 K 2024-11-22T03:39:55,684 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7707745419cb47b79b1297ab3f9881bb, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1732246793312 2024-11-22T03:39:55,685 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] compactions.Compactor(225): Compacting 920b0be291b247aa8d976946b061290c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1732246793312 2024-11-22T03:39:55,685 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] compactions.Compactor(225): Compacting cd4945c96adf4832a8d7671f9f41d297, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1732246793513 2024-11-22T03:39:55,687 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] compactions.Compactor(225): Compacting 7acdd5c62ba64939a5cf0a87243c40ce, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1732246793513 2024-11-22T03:39:55,687 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] compactions.Compactor(225): Compacting cfb481a93dfa49358f7f828d17d57846, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1732246794738 2024-11-22T03:39:55,689 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] compactions.Compactor(225): Compacting f4065c9b2c26424fa79e3683eee8f910, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1732246794738 2024-11-22T03:39:55,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-22T03:39:55,720 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42545 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-22T03:39:55,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4. 2024-11-22T03:39:55,721 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2902): Flushing f970bd9f1e861e2b64a2fb87fb209ac4 3/3 column families, dataSize=114.05 KB heapSize=299.58 KB 2024-11-22T03:39:55,742 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/.tmp/A/2e92634f8b1449f0aa5485b493ef90e7 is 50, key is test_row_0/A:col10/1732246795472/Put/seqid=0 2024-11-22T03:39:55,747 INFO [RS:0;c85114ed5096:42545-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f970bd9f1e861e2b64a2fb87fb209ac4#B#compaction#10 average throughput is 0.66 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-22T03:39:55,748 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/.tmp/B/0f128c9d120046de970579dd7ee0bebd is 50, key is test_row_0/B:col10/1732246795413/Put/seqid=0 2024-11-22T03:39:55,762 INFO [RS:0;c85114ed5096:42545-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f970bd9f1e861e2b64a2fb87fb209ac4#A#compaction#9 average throughput is 0.16 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T03:39:55,763 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/.tmp/A/8ce92063167a442187a26900d790f5e3 is 50, key is test_row_0/A:col10/1732246795413/Put/seqid=0 2024-11-22T03:39:55,779 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073741846_1022 (size=12001) 2024-11-22T03:39:55,781 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=74 (bloomFilter=true), to=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/.tmp/A/2e92634f8b1449f0aa5485b493ef90e7 2024-11-22T03:39:55,795 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073741847_1023 (size=12104) 2024-11-22T03:39:55,811 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/.tmp/B/0f128c9d120046de970579dd7ee0bebd as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/B/0f128c9d120046de970579dd7ee0bebd 2024-11-22T03:39:55,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(8855): Flush requested on f970bd9f1e861e2b64a2fb87fb209ac4 2024-11-22T03:39:55,818 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4. as already flushing 2024-11-22T03:39:55,831 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073741848_1024 (size=12104) 2024-11-22T03:39:55,849 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/.tmp/A/8ce92063167a442187a26900d790f5e3 as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/A/8ce92063167a442187a26900d790f5e3 2024-11-22T03:39:55,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/.tmp/B/e44e1b6aa5ff4495b4856a90fd6c6d42 is 50, key is test_row_0/B:col10/1732246795472/Put/seqid=0 2024-11-22T03:39:55,859 INFO [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f970bd9f1e861e2b64a2fb87fb209ac4/B of f970bd9f1e861e2b64a2fb87fb209ac4 into 0f128c9d120046de970579dd7ee0bebd(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T03:39:55,860 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f970bd9f1e861e2b64a2fb87fb209ac4: 2024-11-22T03:39:55,864 INFO [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., storeName=f970bd9f1e861e2b64a2fb87fb209ac4/B, priority=13, startTime=1732246795669; duration=0sec 2024-11-22T03:39:55,864 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T03:39:55,865 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f970bd9f1e861e2b64a2fb87fb209ac4:B 2024-11-22T03:39:55,865 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T03:39:55,867 INFO [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f970bd9f1e861e2b64a2fb87fb209ac4/A of f970bd9f1e861e2b64a2fb87fb209ac4 into 8ce92063167a442187a26900d790f5e3(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T03:39:55,867 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f970bd9f1e861e2b64a2fb87fb209ac4: 2024-11-22T03:39:55,867 INFO [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., storeName=f970bd9f1e861e2b64a2fb87fb209ac4/A, priority=13, startTime=1732246795664; duration=0sec 2024-11-22T03:39:55,868 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T03:39:55,868 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f970bd9f1e861e2b64a2fb87fb209ac4:A 2024-11-22T03:39:55,870 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T03:39:55,870 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HStore(1541): f970bd9f1e861e2b64a2fb87fb209ac4/C is initiating minor compaction (all files) 2024-11-22T03:39:55,870 INFO [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f970bd9f1e861e2b64a2fb87fb209ac4/C in TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4. 2024-11-22T03:39:55,870 INFO [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/C/8a9d616772a64f4294051324e940a5ca, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/C/eda6acb3909c4de0a95ab22dd66ca079, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/C/e232665be81a42d3b8e5ab310626de53] into tmpdir=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/.tmp, totalSize=35.2 K 2024-11-22T03:39:55,871 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] compactions.Compactor(225): Compacting 8a9d616772a64f4294051324e940a5ca, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1732246793312 2024-11-22T03:39:55,872 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] compactions.Compactor(225): Compacting eda6acb3909c4de0a95ab22dd66ca079, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1732246793513 2024-11-22T03:39:55,874 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] compactions.Compactor(225): Compacting e232665be81a42d3b8e5ab310626de53, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1732246794738 2024-11-22T03:39:55,888 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:39:55,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49612 deadline: 1732246855859, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:39:55,890 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073741849_1025 (size=12001) 2024-11-22T03:39:55,891 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:39:55,891 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:39:55,891 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:39:55,893 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=74 (bloomFilter=true), to=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/.tmp/B/e44e1b6aa5ff4495b4856a90fd6c6d42 2024-11-22T03:39:55,902 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:39:55,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49606 deadline: 1732246855886, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:39:55,903 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:39:55,903 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:39:55,903 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:39:55,903 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:39:55,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49590 deadline: 1732246855886, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:39:55,904 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:39:55,904 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:39:55,904 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:39:55,904 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:39:55,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49582 deadline: 1732246855889, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:39:55,905 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:39:55,905 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:39:55,905 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:39:55,906 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:39:55,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49566 deadline: 1732246855889, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:39:55,906 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:39:55,906 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:39:55,906 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:39:55,917 INFO [RS:0;c85114ed5096:42545-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f970bd9f1e861e2b64a2fb87fb209ac4#C#compaction#13 average throughput is 0.66 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T03:39:55,925 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/.tmp/C/fca97e2fbc9b4dc3a0a8476e5c0efc36 is 50, key is test_row_0/C:col10/1732246795413/Put/seqid=0 2024-11-22T03:39:55,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/.tmp/C/7aac20e45d8d40d6a4db4a305f6d6470 is 50, key is test_row_0/C:col10/1732246795472/Put/seqid=0 2024-11-22T03:39:55,942 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073741850_1026 (size=12104) 2024-11-22T03:39:55,960 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/.tmp/C/fca97e2fbc9b4dc3a0a8476e5c0efc36 as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/C/fca97e2fbc9b4dc3a0a8476e5c0efc36 2024-11-22T03:39:55,976 INFO [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f970bd9f1e861e2b64a2fb87fb209ac4/C of f970bd9f1e861e2b64a2fb87fb209ac4 into fca97e2fbc9b4dc3a0a8476e5c0efc36(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T03:39:55,976 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f970bd9f1e861e2b64a2fb87fb209ac4: 2024-11-22T03:39:55,976 INFO [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., storeName=f970bd9f1e861e2b64a2fb87fb209ac4/C, priority=13, startTime=1732246795669; duration=0sec 2024-11-22T03:39:55,976 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T03:39:55,976 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f970bd9f1e861e2b64a2fb87fb209ac4:C 2024-11-22T03:39:55,992 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073741851_1027 (size=12001) 2024-11-22T03:39:56,000 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:39:56,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49612 deadline: 1732246855999, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:39:56,002 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:39:56,002 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:39:56,002 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:39:56,017 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:39:56,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49590 deadline: 1732246856010, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:39:56,018 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:39:56,018 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:39:56,018 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:39:56,021 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:39:56,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49582 deadline: 1732246856011, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:39:56,023 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:39:56,023 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:39:56,023 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:39:56,023 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:39:56,024 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:39:56,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49606 deadline: 1732246856011, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:39:56,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49566 deadline: 1732246856011, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:39:56,025 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:39:56,025 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:39:56,025 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:39:56,025 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:39:56,025 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:39:56,025 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:39:56,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-22T03:39:56,046 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-22T03:39:56,189 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-22T03:39:56,189 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-11-22T03:39:56,192 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-22T03:39:56,192 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-11-22T03:39:56,192 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-22T03:39:56,192 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-11-22T03:39:56,194 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-22T03:39:56,194 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-11-22T03:39:56,214 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:39:56,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49612 deadline: 1732246856209, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:39:56,215 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:39:56,215 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:39:56,215 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:39:56,230 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:39:56,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49582 deadline: 1732246856229, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:39:56,231 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:39:56,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49566 deadline: 1732246856229, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:39:56,231 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:39:56,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49590 deadline: 1732246856229, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:39:56,232 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:39:56,232 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:39:56,232 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:39:56,233 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:39:56,233 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:39:56,233 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:39:56,233 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:39:56,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49606 deadline: 1732246856230, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:39:56,233 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:39:56,233 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:39:56,234 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:39:56,237 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:39:56,237 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:39:56,237 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:39:56,393 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=74 (bloomFilter=true), to=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/.tmp/C/7aac20e45d8d40d6a4db4a305f6d6470 2024-11-22T03:39:56,410 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/.tmp/A/2e92634f8b1449f0aa5485b493ef90e7 as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/A/2e92634f8b1449f0aa5485b493ef90e7 2024-11-22T03:39:56,425 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/A/2e92634f8b1449f0aa5485b493ef90e7, entries=150, sequenceid=74, filesize=11.7 K 2024-11-22T03:39:56,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/.tmp/B/e44e1b6aa5ff4495b4856a90fd6c6d42 as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/B/e44e1b6aa5ff4495b4856a90fd6c6d42 2024-11-22T03:39:56,441 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/B/e44e1b6aa5ff4495b4856a90fd6c6d42, entries=150, sequenceid=74, filesize=11.7 K 2024-11-22T03:39:56,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/.tmp/C/7aac20e45d8d40d6a4db4a305f6d6470 as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/C/7aac20e45d8d40d6a4db4a305f6d6470 2024-11-22T03:39:56,462 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/C/7aac20e45d8d40d6a4db4a305f6d6470, entries=150, sequenceid=74, filesize=11.7 K 2024-11-22T03:39:56,465 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~299.53 KB/306720, currentSize=93.93 KB/96180 for f970bd9f1e861e2b64a2fb87fb209ac4 in 745ms, sequenceid=74, compaction requested=false 2024-11-22T03:39:56,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2603): Flush status journal for f970bd9f1e861e2b64a2fb87fb209ac4: 2024-11-22T03:39:56,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4. 2024-11-22T03:39:56,466 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=10 2024-11-22T03:39:56,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42627 {}] master.HMaster(4169): Remote procedure done, pid=10 2024-11-22T03:39:56,477 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=10, resume processing ppid=9 2024-11-22T03:39:56,477 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0630 sec 2024-11-22T03:39:56,482 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees in 1.0790 sec 2024-11-22T03:39:56,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(8855): Flush requested on f970bd9f1e861e2b64a2fb87fb209ac4 2024-11-22T03:39:56,523 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-22T03:39:56,523 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f970bd9f1e861e2b64a2fb87fb209ac4 3/3 column families, dataSize=100.63 KB heapSize=264.42 KB 2024-11-22T03:39:56,533 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/.tmp/A/05c1c9a8c4a54140be86638ea9b9e584 is 50, key is test_row_0/A:col10/1732246795885/Put/seqid=0 2024-11-22T03:39:56,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-22T03:39:56,541 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-22T03:39:56,541 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42627 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-22T03:39:56,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42627 {}] procedure2.ProcedureExecutor(1139): Stored pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees 2024-11-22T03:39:56,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-22T03:39:56,545 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-22T03:39:56,546 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-22T03:39:56,547 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=11, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-22T03:39:56,571 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073741852_1028 (size=12001) 2024-11-22T03:39:56,572 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=95 (bloomFilter=true), to=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/.tmp/A/05c1c9a8c4a54140be86638ea9b9e584 2024-11-22T03:39:56,577 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:39:56,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49606 deadline: 1732246856567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:39:56,578 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:39:56,578 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:39:56,578 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:39:56,580 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:39:56,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49612 deadline: 1732246856568, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:39:56,581 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:39:56,581 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:39:56,581 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:39:56,581 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:39:56,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49566 deadline: 1732246856572, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:39:56,582 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:39:56,582 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:39:56,582 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:39:56,582 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:39:56,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49590 deadline: 1732246856572, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:39:56,583 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:39:56,583 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:39:56,583 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:39:56,583 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:39:56,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49582 deadline: 1732246856576, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:39:56,585 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:39:56,585 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:39:56,585 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:39:56,608 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/.tmp/B/8431005a11e742fdad7db991e4bb7edf is 50, key is test_row_0/B:col10/1732246795885/Put/seqid=0 2024-11-22T03:39:56,628 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073741853_1029 (size=12001) 2024-11-22T03:39:56,630 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=95 (bloomFilter=true), to=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/.tmp/B/8431005a11e742fdad7db991e4bb7edf 2024-11-22T03:39:56,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-22T03:39:56,654 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/.tmp/C/4bf36c6e00ea49f5a9436984eeb27813 is 50, key is test_row_0/C:col10/1732246795885/Put/seqid=0 2024-11-22T03:39:56,679 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073741854_1030 (size=12001) 2024-11-22T03:39:56,682 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=95 (bloomFilter=true), to=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/.tmp/C/4bf36c6e00ea49f5a9436984eeb27813 2024-11-22T03:39:56,691 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:39:56,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49612 deadline: 1732246856690, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:39:56,692 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:39:56,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49606 deadline: 1732246856690, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:39:56,693 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:39:56,692 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:39:56,693 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:39:56,693 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:39:56,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49566 deadline: 1732246856690, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:39:56,693 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:39:56,693 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:39:56,693 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:39:56,693 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:39:56,693 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:39:56,693 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:39:56,694 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:39:56,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49590 deadline: 1732246856690, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:39:56,694 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:39:56,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49582 deadline: 1732246856691, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:39:56,694 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:39:56,694 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:39:56,694 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:39:56,695 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:39:56,695 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:39:56,695 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:39:56,700 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42545 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-22T03:39:56,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4. 2024-11-22T03:39:56,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4. as already flushing 2024-11-22T03:39:56,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4. 2024-11-22T03:39:56,701 ERROR [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => f970bd9f1e861e2b64a2fb87fb209ac4, NAME => 'TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:39:56,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => f970bd9f1e861e2b64a2fb87fb209ac4, NAME => 'TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:39:56,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42627 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f970bd9f1e861e2b64a2fb87fb209ac4, NAME => 'TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f970bd9f1e861e2b64a2fb87fb209ac4, NAME => 'TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:39:56,711 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/.tmp/A/05c1c9a8c4a54140be86638ea9b9e584 as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/A/05c1c9a8c4a54140be86638ea9b9e584 2024-11-22T03:39:56,728 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/A/05c1c9a8c4a54140be86638ea9b9e584, entries=150, sequenceid=95, filesize=11.7 K 2024-11-22T03:39:56,731 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/.tmp/B/8431005a11e742fdad7db991e4bb7edf as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/B/8431005a11e742fdad7db991e4bb7edf 2024-11-22T03:39:56,757 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/B/8431005a11e742fdad7db991e4bb7edf, entries=150, sequenceid=95, filesize=11.7 K 2024-11-22T03:39:56,766 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/.tmp/C/4bf36c6e00ea49f5a9436984eeb27813 as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/C/4bf36c6e00ea49f5a9436984eeb27813 2024-11-22T03:39:56,794 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/C/4bf36c6e00ea49f5a9436984eeb27813, entries=150, sequenceid=95, filesize=11.7 K 2024-11-22T03:39:56,796 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~264.38 KB/270720, currentSize=100.63 KB/103050 for f970bd9f1e861e2b64a2fb87fb209ac4 in 273ms, sequenceid=95, compaction requested=true 2024-11-22T03:39:56,797 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f970bd9f1e861e2b64a2fb87fb209ac4: 2024-11-22T03:39:56,797 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f970bd9f1e861e2b64a2fb87fb209ac4:A, priority=-2147483648, current under compaction store size is 1 2024-11-22T03:39:56,797 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T03:39:56,797 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T03:39:56,797 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f970bd9f1e861e2b64a2fb87fb209ac4:B, priority=-2147483648, current under compaction store size is 2 2024-11-22T03:39:56,797 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T03:39:56,797 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f970bd9f1e861e2b64a2fb87fb209ac4:C, priority=-2147483648, current under compaction store size is 3 2024-11-22T03:39:56,797 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-22T03:39:56,797 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T03:39:56,799 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T03:39:56,799 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HStore(1541): f970bd9f1e861e2b64a2fb87fb209ac4/A is initiating minor compaction (all files) 2024-11-22T03:39:56,799 INFO [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f970bd9f1e861e2b64a2fb87fb209ac4/A in TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4. 2024-11-22T03:39:56,800 INFO [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/A/8ce92063167a442187a26900d790f5e3, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/A/2e92634f8b1449f0aa5485b493ef90e7, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/A/05c1c9a8c4a54140be86638ea9b9e584] into tmpdir=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/.tmp, totalSize=35.3 K 2024-11-22T03:39:56,803 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T03:39:56,803 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.HStore(1541): f970bd9f1e861e2b64a2fb87fb209ac4/B is initiating minor compaction (all files) 2024-11-22T03:39:56,803 INFO [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f970bd9f1e861e2b64a2fb87fb209ac4/B in TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4. 2024-11-22T03:39:56,804 INFO [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/B/0f128c9d120046de970579dd7ee0bebd, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/B/e44e1b6aa5ff4495b4856a90fd6c6d42, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/B/8431005a11e742fdad7db991e4bb7edf] into tmpdir=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/.tmp, totalSize=35.3 K 2024-11-22T03:39:56,805 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] compactions.Compactor(225): Compacting 8ce92063167a442187a26900d790f5e3, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1732246794738 2024-11-22T03:39:56,808 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0f128c9d120046de970579dd7ee0bebd, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1732246794738 2024-11-22T03:39:56,809 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] compactions.Compactor(225): Compacting 2e92634f8b1449f0aa5485b493ef90e7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=74, earliestPutTs=1732246795454 2024-11-22T03:39:56,810 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] compactions.Compactor(225): Compacting e44e1b6aa5ff4495b4856a90fd6c6d42, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=74, earliestPutTs=1732246795454 2024-11-22T03:39:56,811 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] compactions.Compactor(225): Compacting 05c1c9a8c4a54140be86638ea9b9e584, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1732246795881 2024-11-22T03:39:56,813 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8431005a11e742fdad7db991e4bb7edf, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1732246795881 2024-11-22T03:39:56,857 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42545 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-22T03:39:56,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4. 2024-11-22T03:39:56,858 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2902): Flushing f970bd9f1e861e2b64a2fb87fb209ac4 3/3 column families, dataSize=100.63 KB heapSize=264.42 KB 2024-11-22T03:39:56,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-22T03:39:56,874 INFO [RS:0;c85114ed5096:42545-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f970bd9f1e861e2b64a2fb87fb209ac4#B#compaction#18 average throughput is 0.82 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T03:39:56,875 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/.tmp/B/1ff2b194d9194e95884acc70c25b145b is 50, key is test_row_0/B:col10/1732246795885/Put/seqid=0 2024-11-22T03:39:56,875 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/.tmp/A/b05283494a0c417baf99a6549ef6981d is 50, key is test_row_0/A:col10/1732246796568/Put/seqid=0 2024-11-22T03:39:56,891 INFO [RS:0;c85114ed5096:42545-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f970bd9f1e861e2b64a2fb87fb209ac4#A#compaction#20 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T03:39:56,892 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/.tmp/A/dc9bd375e9c24a4b959e574cba27aac0 is 50, key is test_row_0/A:col10/1732246795885/Put/seqid=0 2024-11-22T03:39:56,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(8855): Flush requested on f970bd9f1e861e2b64a2fb87fb209ac4 2024-11-22T03:39:56,903 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4. as already flushing 2024-11-22T03:39:56,936 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073741855_1031 (size=12207) 2024-11-22T03:39:56,946 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:39:56,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49566 deadline: 1732246856932, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:39:56,947 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:39:56,947 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:39:56,947 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:39:56,949 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:39:56,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49612 deadline: 1732246856935, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:39:56,950 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:39:56,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49590 deadline: 1732246856936, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:39:56,951 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:39:56,951 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:39:56,951 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:39:56,953 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:39:56,953 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:39:56,953 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:39:56,954 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/.tmp/B/1ff2b194d9194e95884acc70c25b145b as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/B/1ff2b194d9194e95884acc70c25b145b 2024-11-22T03:39:56,955 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:39:56,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49582 deadline: 1732246856942, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:39:56,956 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:39:56,956 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:39:56,956 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:39:56,956 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:39:56,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49606 deadline: 1732246856946, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:39:56,964 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:39:56,964 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:39:56,964 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:39:56,965 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073741856_1032 (size=12001) 2024-11-22T03:39:56,974 INFO [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f970bd9f1e861e2b64a2fb87fb209ac4/B of f970bd9f1e861e2b64a2fb87fb209ac4 into 1ff2b194d9194e95884acc70c25b145b(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T03:39:56,975 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f970bd9f1e861e2b64a2fb87fb209ac4: 2024-11-22T03:39:56,975 INFO [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., storeName=f970bd9f1e861e2b64a2fb87fb209ac4/B, priority=13, startTime=1732246796797; duration=0sec 2024-11-22T03:39:56,975 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T03:39:56,975 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f970bd9f1e861e2b64a2fb87fb209ac4:B 2024-11-22T03:39:56,976 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T03:39:56,980 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T03:39:56,980 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.HStore(1541): f970bd9f1e861e2b64a2fb87fb209ac4/C is initiating minor compaction (all files) 2024-11-22T03:39:56,980 INFO [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f970bd9f1e861e2b64a2fb87fb209ac4/C in TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4. 2024-11-22T03:39:56,981 INFO [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/C/fca97e2fbc9b4dc3a0a8476e5c0efc36, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/C/7aac20e45d8d40d6a4db4a305f6d6470, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/C/4bf36c6e00ea49f5a9436984eeb27813] into tmpdir=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/.tmp, totalSize=35.3 K 2024-11-22T03:39:56,981 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] compactions.Compactor(225): Compacting fca97e2fbc9b4dc3a0a8476e5c0efc36, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1732246794738 2024-11-22T03:39:56,982 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7aac20e45d8d40d6a4db4a305f6d6470, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=74, earliestPutTs=1732246795454 2024-11-22T03:39:56,983 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4bf36c6e00ea49f5a9436984eeb27813, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1732246795881 2024-11-22T03:39:57,000 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073741857_1033 (size=12207) 2024-11-22T03:39:57,012 INFO [RS:0;c85114ed5096:42545-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f970bd9f1e861e2b64a2fb87fb209ac4#C#compaction#21 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T03:39:57,013 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/.tmp/C/50d5d6ce1cb344e799319490c37f9467 is 50, key is test_row_0/C:col10/1732246795885/Put/seqid=0 2024-11-22T03:39:57,024 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/.tmp/A/dc9bd375e9c24a4b959e574cba27aac0 as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/A/dc9bd375e9c24a4b959e574cba27aac0 2024-11-22T03:39:57,038 INFO [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f970bd9f1e861e2b64a2fb87fb209ac4/A of f970bd9f1e861e2b64a2fb87fb209ac4 into dc9bd375e9c24a4b959e574cba27aac0(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T03:39:57,039 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f970bd9f1e861e2b64a2fb87fb209ac4: 2024-11-22T03:39:57,039 INFO [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., storeName=f970bd9f1e861e2b64a2fb87fb209ac4/A, priority=13, startTime=1732246796797; duration=0sec 2024-11-22T03:39:57,039 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T03:39:57,039 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f970bd9f1e861e2b64a2fb87fb209ac4:A 2024-11-22T03:39:57,043 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073741858_1034 (size=12207) 2024-11-22T03:39:57,056 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/.tmp/C/50d5d6ce1cb344e799319490c37f9467 as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/C/50d5d6ce1cb344e799319490c37f9467 2024-11-22T03:39:57,056 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:39:57,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49566 deadline: 1732246857050, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:39:57,057 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:39:57,058 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:39:57,058 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:39:57,059 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:39:57,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49612 deadline: 1732246857059, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:39:57,061 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:39:57,061 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:39:57,061 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:39:57,063 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:39:57,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49582 deadline: 1732246857059, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:39:57,064 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:39:57,064 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:39:57,064 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:39:57,064 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:39:57,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49590 deadline: 1732246857059, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:39:57,070 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:39:57,070 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:39:57,070 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:39:57,075 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:39:57,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49606 deadline: 1732246857069, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:39:57,077 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:39:57,077 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:39:57,077 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:39:57,079 INFO [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f970bd9f1e861e2b64a2fb87fb209ac4/C of f970bd9f1e861e2b64a2fb87fb209ac4 into 50d5d6ce1cb344e799319490c37f9467(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T03:39:57,079 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f970bd9f1e861e2b64a2fb87fb209ac4: 2024-11-22T03:39:57,079 INFO [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., storeName=f970bd9f1e861e2b64a2fb87fb209ac4/C, priority=13, startTime=1732246796797; duration=0sec 2024-11-22T03:39:57,079 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T03:39:57,079 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f970bd9f1e861e2b64a2fb87fb209ac4:C 2024-11-22T03:39:57,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-22T03:39:57,271 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:39:57,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49566 deadline: 1732246857269, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:39:57,272 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:39:57,272 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:39:57,272 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:39:57,275 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:39:57,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49582 deadline: 1732246857269, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:39:57,276 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:39:57,276 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:39:57,276 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:39:57,277 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:39:57,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49612 deadline: 1732246857269, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:39:57,277 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:39:57,277 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:39:57,277 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:39:57,280 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:39:57,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49590 deadline: 1732246857279, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:39:57,281 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:39:57,281 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:39:57,281 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:39:57,292 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:39:57,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49606 deadline: 1732246857288, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:39:57,293 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:39:57,293 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:39:57,293 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:39:57,368 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=113 (bloomFilter=true), to=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/.tmp/A/b05283494a0c417baf99a6549ef6981d 2024-11-22T03:39:57,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/.tmp/B/5c73ef8267f0487780508fca54b319af is 50, key is test_row_0/B:col10/1732246796568/Put/seqid=0 2024-11-22T03:39:57,470 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073741859_1035 (size=12001) 2024-11-22T03:39:57,473 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=113 (bloomFilter=true), to=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/.tmp/B/5c73ef8267f0487780508fca54b319af 2024-11-22T03:39:57,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/.tmp/C/bcd10c89b3cd4c72a3058f5feace7274 is 50, key is test_row_0/C:col10/1732246796568/Put/seqid=0 2024-11-22T03:39:57,547 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073741860_1036 (size=12001) 2024-11-22T03:39:57,548 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=113 (bloomFilter=true), to=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/.tmp/C/bcd10c89b3cd4c72a3058f5feace7274 2024-11-22T03:39:57,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/.tmp/A/b05283494a0c417baf99a6549ef6981d as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/A/b05283494a0c417baf99a6549ef6981d 2024-11-22T03:39:57,576 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/A/b05283494a0c417baf99a6549ef6981d, entries=150, sequenceid=113, filesize=11.7 K 2024-11-22T03:39:57,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/.tmp/B/5c73ef8267f0487780508fca54b319af as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/B/5c73ef8267f0487780508fca54b319af 2024-11-22T03:39:57,583 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:39:57,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49566 deadline: 1732246857579, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:39:57,584 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:39:57,584 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:39:57,585 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:39:57,589 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/B/5c73ef8267f0487780508fca54b319af, entries=150, sequenceid=113, filesize=11.7 K 2024-11-22T03:39:57,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/.tmp/C/bcd10c89b3cd4c72a3058f5feace7274 as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/C/bcd10c89b3cd4c72a3058f5feace7274 2024-11-22T03:39:57,597 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:39:57,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49582 deadline: 1732246857589, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:39:57,599 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:39:57,599 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:39:57,599 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:39:57,601 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:39:57,601 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:39:57,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49590 deadline: 1732246857590, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:39:57,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49612 deadline: 1732246857589, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:39:57,602 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:39:57,602 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:39:57,602 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:39:57,602 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:39:57,602 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:39:57,602 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:39:57,603 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:39:57,604 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/C/bcd10c89b3cd4c72a3058f5feace7274, entries=150, sequenceid=113, filesize=11.7 K 2024-11-22T03:39:57,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49606 deadline: 1732246857600, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:39:57,604 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:39:57,604 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:39:57,605 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:39:57,606 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~264.38 KB/270720, currentSize=100.63 KB/103050 for f970bd9f1e861e2b64a2fb87fb209ac4 in 747ms, sequenceid=113, compaction requested=false 2024-11-22T03:39:57,607 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2603): Flush status journal for f970bd9f1e861e2b64a2fb87fb209ac4: 2024-11-22T03:39:57,607 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4. 2024-11-22T03:39:57,607 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=12 2024-11-22T03:39:57,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42627 {}] master.HMaster(4169): Remote procedure done, pid=12 2024-11-22T03:39:57,614 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=11 2024-11-22T03:39:57,614 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0630 sec 2024-11-22T03:39:57,624 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees in 1.0750 sec 2024-11-22T03:39:57,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-22T03:39:57,679 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-22T03:39:57,680 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42627 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-22T03:39:57,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42627 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees 2024-11-22T03:39:57,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-22T03:39:57,686 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-22T03:39:57,689 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-22T03:39:57,689 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-22T03:39:57,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-22T03:39:57,844 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42545 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-22T03:39:57,844 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4. 2024-11-22T03:39:57,845 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2902): Flushing f970bd9f1e861e2b64a2fb87fb209ac4 3/3 column families, dataSize=100.63 KB heapSize=264.42 KB 2024-11-22T03:39:57,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/.tmp/A/20e630d42fc848cd9efb0e5638edf4d8 is 50, key is test_row_0/A:col10/1732246796931/Put/seqid=0 2024-11-22T03:39:57,883 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073741861_1037 (size=12051) 2024-11-22T03:39:57,885 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=134 (bloomFilter=true), to=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/.tmp/A/20e630d42fc848cd9efb0e5638edf4d8 2024-11-22T03:39:57,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/.tmp/B/a7e25a69ac3140e0a87e1b8c0a58c763 is 50, key is test_row_0/B:col10/1732246796931/Put/seqid=0 2024-11-22T03:39:57,971 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073741862_1038 (size=12051) 2024-11-22T03:39:57,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-22T03:39:58,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(8855): Flush requested on f970bd9f1e861e2b64a2fb87fb209ac4 2024-11-22T03:39:58,091 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4. as already flushing 2024-11-22T03:39:58,131 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:39:58,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49606 deadline: 1732246858123, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:39:58,133 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:39:58,133 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:39:58,133 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:39:58,133 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:39:58,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49590 deadline: 1732246858125, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:39:58,134 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:39:58,134 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:39:58,134 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:39:58,134 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:39:58,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49582 deadline: 1732246858127, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:39:58,134 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:39:58,135 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:39:58,135 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:39:58,135 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:39:58,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49612 deadline: 1732246858129, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:39:58,136 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:39:58,136 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:39:58,136 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:39:58,137 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:39:58,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49566 deadline: 1732246858132, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:39:58,138 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:39:58,138 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:39:58,138 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:39:58,243 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:39:58,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49582 deadline: 1732246858240, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:39:58,244 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:39:58,243 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:39:58,244 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:39:58,244 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:39:58,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49612 deadline: 1732246858240, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:39:58,244 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:39:58,244 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:39:58,244 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:39:58,244 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:39:58,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49590 deadline: 1732246858240, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:39:58,244 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:39:58,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49606 deadline: 1732246858240, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:39:58,245 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:39:58,245 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:39:58,245 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:39:58,245 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:39:58,245 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:39:58,245 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:39:58,250 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:39:58,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49566 deadline: 1732246858249, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:39:58,250 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:39:58,251 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:39:58,251 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:39:58,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-22T03:39:58,373 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=134 (bloomFilter=true), to=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/.tmp/B/a7e25a69ac3140e0a87e1b8c0a58c763 2024-11-22T03:39:58,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/.tmp/C/fe675d3c14a347b89b6378844726f367 is 50, key is test_row_0/C:col10/1732246796931/Put/seqid=0 2024-11-22T03:39:58,407 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073741863_1039 (size=12051) 2024-11-22T03:39:58,414 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=134 (bloomFilter=true), to=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/.tmp/C/fe675d3c14a347b89b6378844726f367 2024-11-22T03:39:58,425 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/.tmp/A/20e630d42fc848cd9efb0e5638edf4d8 as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/A/20e630d42fc848cd9efb0e5638edf4d8 2024-11-22T03:39:58,434 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/A/20e630d42fc848cd9efb0e5638edf4d8, entries=150, sequenceid=134, filesize=11.8 K 2024-11-22T03:39:58,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/.tmp/B/a7e25a69ac3140e0a87e1b8c0a58c763 as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/B/a7e25a69ac3140e0a87e1b8c0a58c763 2024-11-22T03:39:58,456 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:39:58,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49582 deadline: 1732246858449, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:39:58,457 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:39:58,457 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:39:58,457 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:39:58,457 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:39:58,457 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:39:58,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49606 deadline: 1732246858450, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:39:58,458 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:39:58,458 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:39:58,458 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:39:58,458 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:39:58,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49590 deadline: 1732246858450, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:39:58,459 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/B/a7e25a69ac3140e0a87e1b8c0a58c763, entries=150, sequenceid=134, filesize=11.8 K 2024-11-22T03:39:58,462 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:39:58,462 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:39:58,462 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:39:58,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49612 deadline: 1732246858450, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:39:58,465 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:39:58,466 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:39:58,466 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:39:58,467 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:39:58,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49566 deadline: 1732246858459, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:39:58,468 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:39:58,468 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:39:58,468 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:39:58,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/.tmp/C/fe675d3c14a347b89b6378844726f367 as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/C/fe675d3c14a347b89b6378844726f367 2024-11-22T03:39:58,480 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/C/fe675d3c14a347b89b6378844726f367, entries=150, sequenceid=134, filesize=11.8 K 2024-11-22T03:39:58,482 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~264.38 KB/270720, currentSize=100.63 KB/103050 for f970bd9f1e861e2b64a2fb87fb209ac4 in 637ms, sequenceid=134, compaction requested=true 2024-11-22T03:39:58,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2603): Flush status journal for f970bd9f1e861e2b64a2fb87fb209ac4: 2024-11-22T03:39:58,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4. 2024-11-22T03:39:58,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=14 2024-11-22T03:39:58,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42627 {}] master.HMaster(4169): Remote procedure done, pid=14 2024-11-22T03:39:58,488 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13 2024-11-22T03:39:58,489 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 796 msec 2024-11-22T03:39:58,493 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees in 810 msec 2024-11-22T03:39:58,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(8855): Flush requested on f970bd9f1e861e2b64a2fb87fb209ac4 2024-11-22T03:39:58,775 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-22T03:39:58,776 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f970bd9f1e861e2b64a2fb87fb209ac4 3/3 column families, dataSize=114.05 KB heapSize=299.58 KB 2024-11-22T03:39:58,788 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/.tmp/A/36f4ff699dd84da1913edc79545b1f8f is 50, key is test_row_0/A:col10/1732246798772/Put/seqid=0 2024-11-22T03:39:58,810 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073741864_1040 (size=12151) 2024-11-22T03:39:58,811 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=155 (bloomFilter=true), to=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/.tmp/A/36f4ff699dd84da1913edc79545b1f8f 2024-11-22T03:39:58,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-22T03:39:58,819 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-22T03:39:58,820 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42627 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-22T03:39:58,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42627 {}] procedure2.ProcedureExecutor(1139): Stored pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees 2024-11-22T03:39:58,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-22T03:39:58,823 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-22T03:39:58,824 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-22T03:39:58,824 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=16, ppid=15, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-22T03:39:58,827 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/.tmp/B/d9f9f6631f4f4fb3862749634fca533f is 50, key is test_row_0/B:col10/1732246798772/Put/seqid=0 2024-11-22T03:39:58,836 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073741865_1041 (size=12151) 2024-11-22T03:39:58,864 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:39:58,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49590 deadline: 1732246858807, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:39:58,865 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:39:58,865 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:39:58,865 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:39:58,869 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:39:58,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49612 deadline: 1732246858831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:39:58,870 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:39:58,870 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:39:58,870 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:39:58,871 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:39:58,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49606 deadline: 1732246858864, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:39:58,871 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:39:58,871 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:39:58,871 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:39:58,872 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:39:58,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49566 deadline: 1732246858864, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:39:58,872 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:39:58,872 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:39:58,873 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:39:58,873 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:39:58,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49582 deadline: 1732246858865, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:39:58,873 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:39:58,873 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:39:58,873 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:39:58,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-22T03:39:58,969 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:39:58,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49590 deadline: 1732246858968, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:39:58,970 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:39:58,970 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:39:58,970 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:39:58,976 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42545 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-22T03:39:58,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4. 2024-11-22T03:39:58,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4. as already flushing 2024-11-22T03:39:58,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4. 2024-11-22T03:39:58,977 ERROR [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => f970bd9f1e861e2b64a2fb87fb209ac4, NAME => 'TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:39:58,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => f970bd9f1e861e2b64a2fb87fb209ac4, NAME => 'TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:39:58,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42627 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f970bd9f1e861e2b64a2fb87fb209ac4, NAME => 'TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f970bd9f1e861e2b64a2fb87fb209ac4, NAME => 'TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:39:58,981 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:39:58,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49612 deadline: 1732246858979, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:39:58,982 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:39:58,982 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:39:58,982 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:39:58,982 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:39:58,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49606 deadline: 1732246858979, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:39:58,983 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:39:58,983 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:39:58,983 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:39:58,983 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:39:58,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49566 deadline: 1732246858979, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:39:58,984 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:39:58,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49582 deadline: 1732246858980, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:39:58,985 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:39:58,985 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:39:58,985 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:39:58,985 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:39:58,985 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:39:58,985 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:39:59,131 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42545 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-22T03:39:59,131 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4. 2024-11-22T03:39:59,132 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4. as already flushing 2024-11-22T03:39:59,132 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4. 2024-11-22T03:39:59,132 ERROR [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => f970bd9f1e861e2b64a2fb87fb209ac4, NAME => 'TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:39:59,132 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => f970bd9f1e861e2b64a2fb87fb209ac4, NAME => 'TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:39:59,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42627 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f970bd9f1e861e2b64a2fb87fb209ac4, NAME => 'TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f970bd9f1e861e2b64a2fb87fb209ac4, NAME => 'TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:39:59,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-22T03:39:59,183 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:39:59,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49590 deadline: 1732246859181, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:39:59,184 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:39:59,184 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:39:59,184 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:39:59,191 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:39:59,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49612 deadline: 1732246859188, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:39:59,192 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:39:59,192 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:39:59,192 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:39:59,192 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:39:59,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49606 deadline: 1732246859189, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:39:59,192 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:39:59,193 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:39:59,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49566 deadline: 1732246859189, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:39:59,193 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:39:59,193 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:39:59,193 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:39:59,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49582 deadline: 1732246859189, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:39:59,193 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:39:59,193 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:39:59,193 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:39:59,193 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:39:59,193 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:39:59,194 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:39:59,238 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=155 (bloomFilter=true), to=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/.tmp/B/d9f9f6631f4f4fb3862749634fca533f 2024-11-22T03:39:59,253 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/.tmp/C/515908705c6f445bb08d4e794c7576ac is 50, key is test_row_0/C:col10/1732246798772/Put/seqid=0 2024-11-22T03:39:59,268 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073741866_1042 (size=12151) 2024-11-22T03:39:59,273 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=155 (bloomFilter=true), to=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/.tmp/C/515908705c6f445bb08d4e794c7576ac 2024-11-22T03:39:59,286 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/.tmp/A/36f4ff699dd84da1913edc79545b1f8f as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/A/36f4ff699dd84da1913edc79545b1f8f 2024-11-22T03:39:59,286 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42545 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-22T03:39:59,287 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4. 2024-11-22T03:39:59,287 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4. as already flushing 2024-11-22T03:39:59,287 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4. 2024-11-22T03:39:59,287 ERROR [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => f970bd9f1e861e2b64a2fb87fb209ac4, NAME => 'TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:39:59,287 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => f970bd9f1e861e2b64a2fb87fb209ac4, NAME => 'TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:39:59,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42627 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f970bd9f1e861e2b64a2fb87fb209ac4, NAME => 'TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f970bd9f1e861e2b64a2fb87fb209ac4, NAME => 'TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:39:59,304 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/A/36f4ff699dd84da1913edc79545b1f8f, entries=150, sequenceid=155, filesize=11.9 K 2024-11-22T03:39:59,308 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/.tmp/B/d9f9f6631f4f4fb3862749634fca533f as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/B/d9f9f6631f4f4fb3862749634fca533f 2024-11-22T03:39:59,321 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/B/d9f9f6631f4f4fb3862749634fca533f, entries=150, sequenceid=155, filesize=11.9 K 2024-11-22T03:39:59,325 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/.tmp/C/515908705c6f445bb08d4e794c7576ac as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/C/515908705c6f445bb08d4e794c7576ac 2024-11-22T03:39:59,357 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/C/515908705c6f445bb08d4e794c7576ac, entries=150, sequenceid=155, filesize=11.9 K 2024-11-22T03:39:59,358 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~317.11 KB/324720, currentSize=93.93 KB/96180 for f970bd9f1e861e2b64a2fb87fb209ac4 in 583ms, sequenceid=155, compaction requested=true 2024-11-22T03:39:59,358 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f970bd9f1e861e2b64a2fb87fb209ac4: 2024-11-22T03:39:59,358 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f970bd9f1e861e2b64a2fb87fb209ac4:A, priority=-2147483648, current under compaction store size is 1 2024-11-22T03:39:59,359 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-22T03:39:59,359 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T03:39:59,359 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f970bd9f1e861e2b64a2fb87fb209ac4:B, priority=-2147483648, current under compaction store size is 2 2024-11-22T03:39:59,359 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-22T03:39:59,359 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T03:39:59,359 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f970bd9f1e861e2b64a2fb87fb209ac4:C, priority=-2147483648, current under compaction store size is 3 2024-11-22T03:39:59,359 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T03:39:59,364 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48410 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-22T03:39:59,364 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48410 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-22T03:39:59,364 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HStore(1541): f970bd9f1e861e2b64a2fb87fb209ac4/A is initiating minor compaction (all files) 2024-11-22T03:39:59,364 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.HStore(1541): f970bd9f1e861e2b64a2fb87fb209ac4/B is initiating minor compaction (all files) 2024-11-22T03:39:59,364 INFO [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f970bd9f1e861e2b64a2fb87fb209ac4/A in TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4. 2024-11-22T03:39:59,364 INFO [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f970bd9f1e861e2b64a2fb87fb209ac4/B in TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4. 2024-11-22T03:39:59,364 INFO [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/B/1ff2b194d9194e95884acc70c25b145b, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/B/5c73ef8267f0487780508fca54b319af, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/B/a7e25a69ac3140e0a87e1b8c0a58c763, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/B/d9f9f6631f4f4fb3862749634fca533f] into tmpdir=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/.tmp, totalSize=47.3 K 2024-11-22T03:39:59,364 INFO [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/A/dc9bd375e9c24a4b959e574cba27aac0, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/A/b05283494a0c417baf99a6549ef6981d, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/A/20e630d42fc848cd9efb0e5638edf4d8, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/A/36f4ff699dd84da1913edc79545b1f8f] into tmpdir=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/.tmp, totalSize=47.3 K 2024-11-22T03:39:59,365 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1ff2b194d9194e95884acc70c25b145b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1732246795881 2024-11-22T03:39:59,365 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] compactions.Compactor(225): Compacting dc9bd375e9c24a4b959e574cba27aac0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1732246795881 2024-11-22T03:39:59,366 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5c73ef8267f0487780508fca54b319af, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=113, earliestPutTs=1732246796568 2024-11-22T03:39:59,366 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] compactions.Compactor(225): Compacting b05283494a0c417baf99a6549ef6981d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=113, earliestPutTs=1732246796568 2024-11-22T03:39:59,367 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] compactions.Compactor(225): Compacting a7e25a69ac3140e0a87e1b8c0a58c763, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1732246796922 2024-11-22T03:39:59,367 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] compactions.Compactor(225): Compacting 20e630d42fc848cd9efb0e5638edf4d8, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1732246796922 2024-11-22T03:39:59,368 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] compactions.Compactor(225): Compacting 36f4ff699dd84da1913edc79545b1f8f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1732246798772 2024-11-22T03:39:59,368 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] compactions.Compactor(225): Compacting d9f9f6631f4f4fb3862749634fca533f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1732246798772 2024-11-22T03:39:59,401 INFO [RS:0;c85114ed5096:42545-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f970bd9f1e861e2b64a2fb87fb209ac4#B#compaction#30 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T03:39:59,402 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/.tmp/B/9aa2b44f5aca4b8c990f15b9f83fe55a is 50, key is test_row_0/B:col10/1732246798772/Put/seqid=0 2024-11-22T03:39:59,408 INFO [RS:0;c85114ed5096:42545-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f970bd9f1e861e2b64a2fb87fb209ac4#A#compaction#31 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T03:39:59,408 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/.tmp/A/0a82e43cb2cb41838b61e97b23444822 is 50, key is test_row_0/A:col10/1732246798772/Put/seqid=0 2024-11-22T03:39:59,431 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073741868_1044 (size=12493) 2024-11-22T03:39:59,444 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42545 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-22T03:39:59,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4. 2024-11-22T03:39:59,444 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2902): Flushing f970bd9f1e861e2b64a2fb87fb209ac4 3/3 column families, dataSize=93.93 KB heapSize=246.84 KB 2024-11-22T03:39:59,448 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/.tmp/A/0a82e43cb2cb41838b61e97b23444822 as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/A/0a82e43cb2cb41838b61e97b23444822 2024-11-22T03:39:59,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-22T03:39:59,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/.tmp/A/f7a8dc7598c0453695f04d4f4bdb1c5d is 50, key is test_row_0/A:col10/1732246798813/Put/seqid=0 2024-11-22T03:39:59,459 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073741867_1043 (size=12493) 2024-11-22T03:39:59,465 INFO [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in f970bd9f1e861e2b64a2fb87fb209ac4/A of f970bd9f1e861e2b64a2fb87fb209ac4 into 0a82e43cb2cb41838b61e97b23444822(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T03:39:59,465 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f970bd9f1e861e2b64a2fb87fb209ac4: 2024-11-22T03:39:59,465 INFO [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., storeName=f970bd9f1e861e2b64a2fb87fb209ac4/A, priority=12, startTime=1732246799358; duration=0sec 2024-11-22T03:39:59,465 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T03:39:59,465 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f970bd9f1e861e2b64a2fb87fb209ac4:A 2024-11-22T03:39:59,465 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-22T03:39:59,467 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48410 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-22T03:39:59,468 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HStore(1541): f970bd9f1e861e2b64a2fb87fb209ac4/C is initiating minor compaction (all files) 2024-11-22T03:39:59,468 INFO [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f970bd9f1e861e2b64a2fb87fb209ac4/C in TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4. 2024-11-22T03:39:59,468 INFO [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/C/50d5d6ce1cb344e799319490c37f9467, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/C/bcd10c89b3cd4c72a3058f5feace7274, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/C/fe675d3c14a347b89b6378844726f367, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/C/515908705c6f445bb08d4e794c7576ac] into tmpdir=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/.tmp, totalSize=47.3 K 2024-11-22T03:39:59,469 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] compactions.Compactor(225): Compacting 50d5d6ce1cb344e799319490c37f9467, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1732246795881 2024-11-22T03:39:59,471 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] compactions.Compactor(225): Compacting bcd10c89b3cd4c72a3058f5feace7274, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=113, earliestPutTs=1732246796568 2024-11-22T03:39:59,472 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] compactions.Compactor(225): Compacting fe675d3c14a347b89b6378844726f367, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1732246796922 2024-11-22T03:39:59,473 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] compactions.Compactor(225): Compacting 515908705c6f445bb08d4e794c7576ac, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1732246798772 2024-11-22T03:39:59,492 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073741869_1045 (size=12151) 2024-11-22T03:39:59,492 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4. as already flushing 2024-11-22T03:39:59,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(8855): Flush requested on f970bd9f1e861e2b64a2fb87fb209ac4 2024-11-22T03:39:59,494 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=172 (bloomFilter=true), to=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/.tmp/A/f7a8dc7598c0453695f04d4f4bdb1c5d 2024-11-22T03:39:59,513 INFO [RS:0;c85114ed5096:42545-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f970bd9f1e861e2b64a2fb87fb209ac4#C#compaction#33 average throughput is 1.31 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T03:39:59,514 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/.tmp/C/94b34a12a19745dc85d07bbbabd0d6b3 is 50, key is test_row_0/C:col10/1732246798772/Put/seqid=0 2024-11-22T03:39:59,519 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/.tmp/B/cc3ca2a6e42c49eb85d94b4b08fe9104 is 50, key is test_row_0/B:col10/1732246798813/Put/seqid=0 2024-11-22T03:39:59,521 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073741870_1046 (size=12493) 2024-11-22T03:39:59,531 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073741871_1047 (size=12151) 2024-11-22T03:39:59,531 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/.tmp/C/94b34a12a19745dc85d07bbbabd0d6b3 as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/C/94b34a12a19745dc85d07bbbabd0d6b3 2024-11-22T03:39:59,532 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=172 (bloomFilter=true), to=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/.tmp/B/cc3ca2a6e42c49eb85d94b4b08fe9104 2024-11-22T03:39:59,544 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/.tmp/C/d6dbee770d1146b8bf3a9dc44312317f is 50, key is test_row_0/C:col10/1732246798813/Put/seqid=0 2024-11-22T03:39:59,548 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073741872_1048 (size=12151) 2024-11-22T03:39:59,549 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=172 (bloomFilter=true), to=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/.tmp/C/d6dbee770d1146b8bf3a9dc44312317f 2024-11-22T03:39:59,556 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/.tmp/A/f7a8dc7598c0453695f04d4f4bdb1c5d as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/A/f7a8dc7598c0453695f04d4f4bdb1c5d 2024-11-22T03:39:59,557 INFO [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in f970bd9f1e861e2b64a2fb87fb209ac4/C of f970bd9f1e861e2b64a2fb87fb209ac4 into 94b34a12a19745dc85d07bbbabd0d6b3(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T03:39:59,557 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f970bd9f1e861e2b64a2fb87fb209ac4: 2024-11-22T03:39:59,558 INFO [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., storeName=f970bd9f1e861e2b64a2fb87fb209ac4/C, priority=12, startTime=1732246799359; duration=0sec 2024-11-22T03:39:59,558 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T03:39:59,558 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f970bd9f1e861e2b64a2fb87fb209ac4:C 2024-11-22T03:39:59,565 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/A/f7a8dc7598c0453695f04d4f4bdb1c5d, entries=150, sequenceid=172, filesize=11.9 K 2024-11-22T03:39:59,567 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/.tmp/B/cc3ca2a6e42c49eb85d94b4b08fe9104 as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/B/cc3ca2a6e42c49eb85d94b4b08fe9104 2024-11-22T03:39:59,577 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/B/cc3ca2a6e42c49eb85d94b4b08fe9104, entries=150, sequenceid=172, filesize=11.9 K 2024-11-22T03:39:59,578 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:39:59,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49566 deadline: 1732246859572, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:39:59,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/.tmp/C/d6dbee770d1146b8bf3a9dc44312317f as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/C/d6dbee770d1146b8bf3a9dc44312317f 2024-11-22T03:39:59,579 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:39:59,579 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:39:59,579 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:39:59,582 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:39:59,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49590 deadline: 1732246859576, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:39:59,583 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:39:59,583 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:39:59,583 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:39:59,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49582 deadline: 1732246859576, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:39:59,583 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:39:59,584 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:39:59,584 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:39:59,584 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:39:59,584 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:39:59,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49612 deadline: 1732246859576, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:39:59,585 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:39:59,585 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:39:59,585 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:39:59,585 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:39:59,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49606 deadline: 1732246859579, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:39:59,586 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:39:59,586 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:39:59,586 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:39:59,588 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/C/d6dbee770d1146b8bf3a9dc44312317f, entries=150, sequenceid=172, filesize=11.9 K 2024-11-22T03:39:59,590 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~246.80 KB/252720, currentSize=107.34 KB/109920 for f970bd9f1e861e2b64a2fb87fb209ac4 in 145ms, sequenceid=172, compaction requested=false 2024-11-22T03:39:59,590 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2603): Flush status journal for f970bd9f1e861e2b64a2fb87fb209ac4: 2024-11-22T03:39:59,590 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4. 2024-11-22T03:39:59,590 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=16 2024-11-22T03:39:59,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42627 {}] master.HMaster(4169): Remote procedure done, pid=16 2024-11-22T03:39:59,597 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=16, resume processing ppid=15 2024-11-22T03:39:59,597 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 769 msec 2024-11-22T03:39:59,602 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees in 778 msec 2024-11-22T03:39:59,693 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-22T03:39:59,694 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f970bd9f1e861e2b64a2fb87fb209ac4 3/3 column families, dataSize=114.05 KB heapSize=299.58 KB 2024-11-22T03:39:59,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(8855): Flush requested on f970bd9f1e861e2b64a2fb87fb209ac4 2024-11-22T03:39:59,712 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/.tmp/A/05035c1f745345c3b99912723bf86b20 is 50, key is test_row_0/A:col10/1732246799576/Put/seqid=0 2024-11-22T03:39:59,724 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:39:59,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49582 deadline: 1732246859718, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:39:59,726 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:39:59,726 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:39:59,726 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:39:59,727 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:39:59,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49566 deadline: 1732246859721, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:39:59,729 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:39:59,729 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:39:59,729 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:39:59,733 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:39:59,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49612 deadline: 1732246859724, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:39:59,734 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:39:59,734 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:39:59,734 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:39:59,734 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:39:59,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49590 deadline: 1732246859725, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:39:59,735 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:39:59,735 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:39:59,735 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:39:59,735 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:39:59,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49606 deadline: 1732246859725, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:39:59,736 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:39:59,736 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:39:59,736 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:39:59,747 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073741873_1049 (size=14541) 2024-11-22T03:39:59,748 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=194 (bloomFilter=true), to=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/.tmp/A/05035c1f745345c3b99912723bf86b20 2024-11-22T03:39:59,765 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/.tmp/B/c976b62d8c01408cb8e1e55b628c7e95 is 50, key is test_row_0/B:col10/1732246799576/Put/seqid=0 2024-11-22T03:39:59,791 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073741874_1050 (size=12151) 2024-11-22T03:39:59,792 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=194 (bloomFilter=true), to=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/.tmp/B/c976b62d8c01408cb8e1e55b628c7e95 2024-11-22T03:39:59,807 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/.tmp/C/a34d358420cf44e69c2304c411cb8b83 is 50, key is test_row_0/C:col10/1732246799576/Put/seqid=0 2024-11-22T03:39:59,830 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:39:59,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49582 deadline: 1732246859828, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:39:59,831 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:39:59,831 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:39:59,831 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:39:59,840 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:39:59,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49612 deadline: 1732246859839, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:39:59,843 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:39:59,843 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:39:59,843 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:39:59,844 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:39:59,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49606 deadline: 1732246859840, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:39:59,844 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:39:59,844 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:39:59,845 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:39:59,845 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:39:59,845 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:39:59,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49566 deadline: 1732246859840, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:39:59,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49590 deadline: 1732246859840, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:39:59,845 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:39:59,845 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:39:59,845 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:39:59,846 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:39:59,846 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:39:59,846 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:39:59,854 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073741875_1051 (size=12151) 2024-11-22T03:39:59,856 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=194 (bloomFilter=true), to=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/.tmp/C/a34d358420cf44e69c2304c411cb8b83 2024-11-22T03:39:59,875 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/.tmp/B/9aa2b44f5aca4b8c990f15b9f83fe55a as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/B/9aa2b44f5aca4b8c990f15b9f83fe55a 2024-11-22T03:39:59,876 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/.tmp/A/05035c1f745345c3b99912723bf86b20 as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/A/05035c1f745345c3b99912723bf86b20 2024-11-22T03:39:59,889 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/A/05035c1f745345c3b99912723bf86b20, entries=200, sequenceid=194, filesize=14.2 K 2024-11-22T03:39:59,893 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/.tmp/B/c976b62d8c01408cb8e1e55b628c7e95 as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/B/c976b62d8c01408cb8e1e55b628c7e95 2024-11-22T03:39:59,894 INFO [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in f970bd9f1e861e2b64a2fb87fb209ac4/B of f970bd9f1e861e2b64a2fb87fb209ac4 into 9aa2b44f5aca4b8c990f15b9f83fe55a(size=12.2 K), total size for store is 24.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T03:39:59,894 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f970bd9f1e861e2b64a2fb87fb209ac4: 2024-11-22T03:39:59,894 INFO [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., storeName=f970bd9f1e861e2b64a2fb87fb209ac4/B, priority=12, startTime=1732246799359; duration=0sec 2024-11-22T03:39:59,896 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T03:39:59,896 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f970bd9f1e861e2b64a2fb87fb209ac4:B 2024-11-22T03:39:59,914 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/B/c976b62d8c01408cb8e1e55b628c7e95, entries=150, sequenceid=194, filesize=11.9 K 2024-11-22T03:39:59,916 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/.tmp/C/a34d358420cf44e69c2304c411cb8b83 as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/C/a34d358420cf44e69c2304c411cb8b83 2024-11-22T03:39:59,927 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/C/a34d358420cf44e69c2304c411cb8b83, entries=150, sequenceid=194, filesize=11.9 K 2024-11-22T03:39:59,928 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~299.53 KB/306720, currentSize=87.22 KB/89310 for f970bd9f1e861e2b64a2fb87fb209ac4 in 235ms, sequenceid=194, compaction requested=true 2024-11-22T03:39:59,929 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f970bd9f1e861e2b64a2fb87fb209ac4: 2024-11-22T03:39:59,929 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f970bd9f1e861e2b64a2fb87fb209ac4:A, priority=-2147483648, current under compaction store size is 1 2024-11-22T03:39:59,929 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T03:39:59,929 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T03:39:59,929 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f970bd9f1e861e2b64a2fb87fb209ac4:B, priority=-2147483648, current under compaction store size is 2 2024-11-22T03:39:59,929 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T03:39:59,929 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f970bd9f1e861e2b64a2fb87fb209ac4:C, priority=-2147483648, current under compaction store size is 3 2024-11-22T03:39:59,929 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T03:39:59,929 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T03:39:59,931 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39185 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T03:39:59,931 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T03:39:59,931 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.HStore(1541): f970bd9f1e861e2b64a2fb87fb209ac4/A is initiating minor compaction (all files) 2024-11-22T03:39:59,931 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HStore(1541): f970bd9f1e861e2b64a2fb87fb209ac4/B is initiating minor compaction (all files) 2024-11-22T03:39:59,931 INFO [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f970bd9f1e861e2b64a2fb87fb209ac4/A in TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4. 2024-11-22T03:39:59,931 INFO [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f970bd9f1e861e2b64a2fb87fb209ac4/B in TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4. 2024-11-22T03:39:59,931 INFO [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/A/0a82e43cb2cb41838b61e97b23444822, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/A/f7a8dc7598c0453695f04d4f4bdb1c5d, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/A/05035c1f745345c3b99912723bf86b20] into tmpdir=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/.tmp, totalSize=38.3 K 2024-11-22T03:39:59,931 INFO [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/B/9aa2b44f5aca4b8c990f15b9f83fe55a, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/B/cc3ca2a6e42c49eb85d94b4b08fe9104, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/B/c976b62d8c01408cb8e1e55b628c7e95] into tmpdir=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/.tmp, totalSize=35.9 K 2024-11-22T03:39:59,932 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] compactions.Compactor(225): Compacting 9aa2b44f5aca4b8c990f15b9f83fe55a, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1732246798772 2024-11-22T03:39:59,932 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0a82e43cb2cb41838b61e97b23444822, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1732246798772 2024-11-22T03:39:59,932 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] compactions.Compactor(225): Compacting cc3ca2a6e42c49eb85d94b4b08fe9104, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1732246798804 2024-11-22T03:39:59,933 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] compactions.Compactor(225): Compacting f7a8dc7598c0453695f04d4f4bdb1c5d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1732246798804 2024-11-22T03:39:59,933 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] compactions.Compactor(225): Compacting c976b62d8c01408cb8e1e55b628c7e95, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1732246799572 2024-11-22T03:39:59,933 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] compactions.Compactor(225): Compacting 05035c1f745345c3b99912723bf86b20, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1732246799572 2024-11-22T03:39:59,949 INFO [RS:0;c85114ed5096:42545-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f970bd9f1e861e2b64a2fb87fb209ac4#A#compaction#39 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T03:39:59,949 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/.tmp/A/00f669db6b0b4f248e23aea844f17710 is 50, key is test_row_0/A:col10/1732246799576/Put/seqid=0 2024-11-22T03:39:59,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-22T03:39:59,960 INFO [RS:0;c85114ed5096:42545-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f970bd9f1e861e2b64a2fb87fb209ac4#B#compaction#40 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T03:39:59,961 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/.tmp/B/a694ac35e4c94035986bf5f8175a2f16 is 50, key is test_row_0/B:col10/1732246799576/Put/seqid=0 2024-11-22T03:39:59,961 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-22T03:39:59,962 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42627 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-22T03:39:59,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42627 {}] procedure2.ProcedureExecutor(1139): Stored pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees 2024-11-22T03:39:59,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-22T03:39:59,967 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-22T03:39:59,968 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-22T03:39:59,969 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=18, ppid=17, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-22T03:40:00,003 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073741876_1052 (size=12595) 2024-11-22T03:40:00,033 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073741877_1053 (size=12595) 2024-11-22T03:40:00,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(8855): Flush requested on f970bd9f1e861e2b64a2fb87fb209ac4 2024-11-22T03:40:00,042 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-22T03:40:00,042 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f970bd9f1e861e2b64a2fb87fb209ac4 3/3 column families, dataSize=93.93 KB heapSize=246.84 KB 2024-11-22T03:40:00,065 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/.tmp/A/79873c10df23470e91a8370dd7307ef0 is 50, key is test_row_0/A:col10/1732246799723/Put/seqid=0 2024-11-22T03:40:00,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-22T03:40:00,095 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:00,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49606 deadline: 1732246860083, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:00,096 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:00,096 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:00,096 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:40:00,096 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:00,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49612 deadline: 1732246860086, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:00,097 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:00,097 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:00,097 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:40:00,097 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:00,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49566 deadline: 1732246860089, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:00,098 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073741878_1054 (size=14541) 2024-11-22T03:40:00,098 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:00,098 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:00,099 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:40:00,098 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:00,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49582 deadline: 1732246860089, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:00,098 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:00,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49590 deadline: 1732246860092, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:00,099 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:00,099 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:00,099 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:40:00,099 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:00,100 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:00,100 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:40:00,100 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=212 (bloomFilter=true), to=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/.tmp/A/79873c10df23470e91a8370dd7307ef0 2024-11-22T03:40:00,115 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/.tmp/B/b5b38c5af776443aa690171ccc4f2d89 is 50, key is test_row_0/B:col10/1732246799723/Put/seqid=0 2024-11-22T03:40:00,124 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42545 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-22T03:40:00,124 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4. 2024-11-22T03:40:00,124 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4. as already flushing 2024-11-22T03:40:00,124 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4. 2024-11-22T03:40:00,124 ERROR [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => f970bd9f1e861e2b64a2fb87fb209ac4, NAME => 'TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:40:00,124 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => f970bd9f1e861e2b64a2fb87fb209ac4, NAME => 'TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:40:00,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42627 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f970bd9f1e861e2b64a2fb87fb209ac4, NAME => 'TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f970bd9f1e861e2b64a2fb87fb209ac4, NAME => 'TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:40:00,137 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073741879_1055 (size=12151) 2024-11-22T03:40:00,200 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:00,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49606 deadline: 1732246860198, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:00,200 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:00,201 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:00,201 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:40:00,215 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:00,215 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:00,215 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:00,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49590 deadline: 1732246860209, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:00,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49582 deadline: 1732246860209, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:00,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49566 deadline: 1732246860209, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:00,216 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:00,216 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:00,216 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:00,216 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:40:00,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49612 deadline: 1732246860210, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:00,216 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:00,216 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:00,216 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:40:00,217 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:00,217 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:00,217 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:40:00,218 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:00,218 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:00,218 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:40:00,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-22T03:40:00,280 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42545 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-22T03:40:00,280 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4. 2024-11-22T03:40:00,280 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4. as already flushing 2024-11-22T03:40:00,281 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4. 2024-11-22T03:40:00,281 ERROR [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => f970bd9f1e861e2b64a2fb87fb209ac4, NAME => 'TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:40:00,281 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => f970bd9f1e861e2b64a2fb87fb209ac4, NAME => 'TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:40:00,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42627 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f970bd9f1e861e2b64a2fb87fb209ac4, NAME => 'TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f970bd9f1e861e2b64a2fb87fb209ac4, NAME => 'TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:40:00,411 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:00,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49606 deadline: 1732246860408, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:00,412 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:00,412 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:00,412 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:40:00,412 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/.tmp/A/00f669db6b0b4f248e23aea844f17710 as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/A/00f669db6b0b4f248e23aea844f17710 2024-11-22T03:40:00,424 INFO [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f970bd9f1e861e2b64a2fb87fb209ac4/A of f970bd9f1e861e2b64a2fb87fb209ac4 into 00f669db6b0b4f248e23aea844f17710(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T03:40:00,424 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f970bd9f1e861e2b64a2fb87fb209ac4: 2024-11-22T03:40:00,424 INFO [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., storeName=f970bd9f1e861e2b64a2fb87fb209ac4/A, priority=13, startTime=1732246799929; duration=0sec 2024-11-22T03:40:00,424 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T03:40:00,424 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f970bd9f1e861e2b64a2fb87fb209ac4:A 2024-11-22T03:40:00,424 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T03:40:00,425 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:00,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49612 deadline: 1732246860419, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:00,426 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:00,426 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:00,426 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:40:00,427 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T03:40:00,427 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.HStore(1541): f970bd9f1e861e2b64a2fb87fb209ac4/C is initiating minor compaction (all files) 2024-11-22T03:40:00,427 INFO [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f970bd9f1e861e2b64a2fb87fb209ac4/C in TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4. 2024-11-22T03:40:00,428 INFO [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/C/94b34a12a19745dc85d07bbbabd0d6b3, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/C/d6dbee770d1146b8bf3a9dc44312317f, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/C/a34d358420cf44e69c2304c411cb8b83] into tmpdir=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/.tmp, totalSize=35.9 K 2024-11-22T03:40:00,429 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:00,429 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:00,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49590 deadline: 1732246860429, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:00,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49582 deadline: 1732246860429, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:00,430 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:00,430 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:00,430 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:00,430 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:40:00,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49566 deadline: 1732246860429, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:00,430 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:00,430 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:00,430 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:40:00,431 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:00,431 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:00,431 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:40:00,431 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] compactions.Compactor(225): Compacting 94b34a12a19745dc85d07bbbabd0d6b3, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1732246798772 2024-11-22T03:40:00,432 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] compactions.Compactor(225): Compacting d6dbee770d1146b8bf3a9dc44312317f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1732246798804 2024-11-22T03:40:00,433 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] compactions.Compactor(225): Compacting a34d358420cf44e69c2304c411cb8b83, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1732246799572 2024-11-22T03:40:00,436 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42545 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-22T03:40:00,437 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4. 2024-11-22T03:40:00,437 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4. as already flushing 2024-11-22T03:40:00,437 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4. 2024-11-22T03:40:00,437 ERROR [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => f970bd9f1e861e2b64a2fb87fb209ac4, NAME => 'TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:40:00,437 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => f970bd9f1e861e2b64a2fb87fb209ac4, NAME => 'TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:40:00,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42627 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f970bd9f1e861e2b64a2fb87fb209ac4, NAME => 'TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f970bd9f1e861e2b64a2fb87fb209ac4, NAME => 'TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:40:00,449 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/.tmp/B/a694ac35e4c94035986bf5f8175a2f16 as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/B/a694ac35e4c94035986bf5f8175a2f16 2024-11-22T03:40:00,464 INFO [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f970bd9f1e861e2b64a2fb87fb209ac4/B of f970bd9f1e861e2b64a2fb87fb209ac4 into a694ac35e4c94035986bf5f8175a2f16(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T03:40:00,464 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f970bd9f1e861e2b64a2fb87fb209ac4: 2024-11-22T03:40:00,464 INFO [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., storeName=f970bd9f1e861e2b64a2fb87fb209ac4/B, priority=13, startTime=1732246799929; duration=0sec 2024-11-22T03:40:00,465 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T03:40:00,465 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f970bd9f1e861e2b64a2fb87fb209ac4:B 2024-11-22T03:40:00,479 INFO [RS:0;c85114ed5096:42545-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f970bd9f1e861e2b64a2fb87fb209ac4#C#compaction#43 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T03:40:00,480 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/.tmp/C/60e84e1c86ad4cd3aaa3d684cc06a3ad is 50, key is test_row_0/C:col10/1732246799576/Put/seqid=0 2024-11-22T03:40:00,512 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073741880_1056 (size=12595) 2024-11-22T03:40:00,539 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=212 (bloomFilter=true), to=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/.tmp/B/b5b38c5af776443aa690171ccc4f2d89 2024-11-22T03:40:00,585 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/.tmp/C/6fae8a2e9abd4328acd0050de197d16c is 50, key is test_row_0/C:col10/1732246799723/Put/seqid=0 2024-11-22T03:40:00,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-22T03:40:00,591 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42545 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-22T03:40:00,592 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4. 2024-11-22T03:40:00,592 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4. as already flushing 2024-11-22T03:40:00,592 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4. 2024-11-22T03:40:00,592 ERROR [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => f970bd9f1e861e2b64a2fb87fb209ac4, NAME => 'TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:40:00,592 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => f970bd9f1e861e2b64a2fb87fb209ac4, NAME => 'TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:40:00,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42627 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f970bd9f1e861e2b64a2fb87fb209ac4, NAME => 'TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f970bd9f1e861e2b64a2fb87fb209ac4, NAME => 'TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:40:00,626 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073741881_1057 (size=12151) 2024-11-22T03:40:00,628 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=212 (bloomFilter=true), to=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/.tmp/C/6fae8a2e9abd4328acd0050de197d16c 2024-11-22T03:40:00,654 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/.tmp/A/79873c10df23470e91a8370dd7307ef0 as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/A/79873c10df23470e91a8370dd7307ef0 2024-11-22T03:40:00,662 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/A/79873c10df23470e91a8370dd7307ef0, entries=200, sequenceid=212, filesize=14.2 K 2024-11-22T03:40:00,664 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/.tmp/B/b5b38c5af776443aa690171ccc4f2d89 as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/B/b5b38c5af776443aa690171ccc4f2d89 2024-11-22T03:40:00,674 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/B/b5b38c5af776443aa690171ccc4f2d89, entries=150, sequenceid=212, filesize=11.9 K 2024-11-22T03:40:00,675 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/.tmp/C/6fae8a2e9abd4328acd0050de197d16c as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/C/6fae8a2e9abd4328acd0050de197d16c 2024-11-22T03:40:00,684 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/C/6fae8a2e9abd4328acd0050de197d16c, entries=150, sequenceid=212, filesize=11.9 K 2024-11-22T03:40:00,686 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~246.80 KB/252720, currentSize=107.34 KB/109920 for f970bd9f1e861e2b64a2fb87fb209ac4 in 643ms, sequenceid=212, compaction requested=false 2024-11-22T03:40:00,686 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f970bd9f1e861e2b64a2fb87fb209ac4: 2024-11-22T03:40:00,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(8855): Flush requested on f970bd9f1e861e2b64a2fb87fb209ac4 2024-11-22T03:40:00,720 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-22T03:40:00,721 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f970bd9f1e861e2b64a2fb87fb209ac4 3/3 column families, dataSize=114.05 KB heapSize=299.58 KB 2024-11-22T03:40:00,726 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/.tmp/A/ab5718b4e15340b08839c04e41e389eb is 50, key is test_row_0/A:col10/1732246800075/Put/seqid=0 2024-11-22T03:40:00,741 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073741882_1058 (size=14541) 2024-11-22T03:40:00,748 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42545 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-22T03:40:00,748 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4. 2024-11-22T03:40:00,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4. as already flushing 2024-11-22T03:40:00,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4. 2024-11-22T03:40:00,749 ERROR [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => f970bd9f1e861e2b64a2fb87fb209ac4, NAME => 'TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:40:00,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => f970bd9f1e861e2b64a2fb87fb209ac4, NAME => 'TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:40:00,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42627 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f970bd9f1e861e2b64a2fb87fb209ac4, NAME => 'TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f970bd9f1e861e2b64a2fb87fb209ac4, NAME => 'TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:40:00,765 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:00,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49566 deadline: 1732246860751, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:00,766 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:00,766 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:00,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49612 deadline: 1732246860752, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:00,767 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:00,767 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:00,767 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:40:00,766 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:00,769 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:40:00,770 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:00,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49606 deadline: 1732246860763, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:00,771 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:00,771 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:00,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49590 deadline: 1732246860766, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:00,771 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:00,772 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:00,772 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:40:00,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49582 deadline: 1732246860764, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:00,772 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:00,772 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:00,772 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:40:00,773 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:00,773 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:00,773 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:40:00,882 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:00,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49606 deadline: 1732246860879, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:00,882 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:00,882 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:00,882 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:00,883 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:40:00,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49590 deadline: 1732246860879, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:00,883 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:00,883 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:00,883 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:00,883 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:40:00,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49612 deadline: 1732246860880, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:00,883 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:00,883 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:00,883 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:40:00,883 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:00,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49566 deadline: 1732246860880, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:00,884 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:00,884 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:00,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49582 deadline: 1732246860880, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:00,884 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:00,884 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:40:00,885 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:00,885 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:00,885 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:40:00,905 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42545 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-22T03:40:00,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4. 2024-11-22T03:40:00,906 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4. as already flushing 2024-11-22T03:40:00,906 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4. 2024-11-22T03:40:00,906 ERROR [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => f970bd9f1e861e2b64a2fb87fb209ac4, NAME => 'TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:40:00,906 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => f970bd9f1e861e2b64a2fb87fb209ac4, NAME => 'TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:40:00,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42627 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f970bd9f1e861e2b64a2fb87fb209ac4, NAME => 'TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f970bd9f1e861e2b64a2fb87fb209ac4, NAME => 'TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:40:00,923 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/.tmp/C/60e84e1c86ad4cd3aaa3d684cc06a3ad as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/C/60e84e1c86ad4cd3aaa3d684cc06a3ad 2024-11-22T03:40:00,934 INFO [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f970bd9f1e861e2b64a2fb87fb209ac4/C of f970bd9f1e861e2b64a2fb87fb209ac4 into 60e84e1c86ad4cd3aaa3d684cc06a3ad(size=12.3 K), total size for store is 24.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T03:40:00,934 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f970bd9f1e861e2b64a2fb87fb209ac4: 2024-11-22T03:40:00,934 INFO [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., storeName=f970bd9f1e861e2b64a2fb87fb209ac4/C, priority=13, startTime=1732246799929; duration=0sec 2024-11-22T03:40:00,934 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T03:40:00,934 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f970bd9f1e861e2b64a2fb87fb209ac4:C 2024-11-22T03:40:01,061 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42545 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-22T03:40:01,061 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4. 2024-11-22T03:40:01,061 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4. as already flushing 2024-11-22T03:40:01,061 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4. 2024-11-22T03:40:01,061 ERROR [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => f970bd9f1e861e2b64a2fb87fb209ac4, NAME => 'TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:40:01,061 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => f970bd9f1e861e2b64a2fb87fb209ac4, NAME => 'TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:40:01,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42627 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f970bd9f1e861e2b64a2fb87fb209ac4, NAME => 'TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f970bd9f1e861e2b64a2fb87fb209ac4, NAME => 'TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:40:01,090 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:01,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49566 deadline: 1732246861089, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:01,091 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:01,091 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:01,091 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:40:01,091 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:01,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49590 deadline: 1732246861089, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:01,092 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:01,092 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:01,092 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:40:01,092 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:01,092 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:01,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49612 deadline: 1732246861089, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:01,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49582 deadline: 1732246861089, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:01,093 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:01,093 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:01,093 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:01,093 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:01,093 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:40:01,093 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:40:01,093 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:01,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49606 deadline: 1732246861089, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:01,094 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:01,094 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:01,094 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:40:01,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-22T03:40:01,142 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=234 (bloomFilter=true), to=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/.tmp/A/ab5718b4e15340b08839c04e41e389eb 2024-11-22T03:40:01,187 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/.tmp/B/78bf0c831baf4229a20ac284a6e5da48 is 50, key is test_row_0/B:col10/1732246800075/Put/seqid=0 2024-11-22T03:40:01,215 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42545 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-22T03:40:01,215 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4. 2024-11-22T03:40:01,216 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4. as already flushing 2024-11-22T03:40:01,216 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4. 2024-11-22T03:40:01,216 ERROR [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => f970bd9f1e861e2b64a2fb87fb209ac4, NAME => 'TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:40:01,216 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => f970bd9f1e861e2b64a2fb87fb209ac4, NAME => 'TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:40:01,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42627 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f970bd9f1e861e2b64a2fb87fb209ac4, NAME => 'TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f970bd9f1e861e2b64a2fb87fb209ac4, NAME => 'TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:40:01,221 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073741883_1059 (size=12151) 2024-11-22T03:40:01,370 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42545 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-22T03:40:01,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4. 2024-11-22T03:40:01,371 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4. as already flushing 2024-11-22T03:40:01,371 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4. 2024-11-22T03:40:01,371 ERROR [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => f970bd9f1e861e2b64a2fb87fb209ac4, NAME => 'TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:40:01,371 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => f970bd9f1e861e2b64a2fb87fb209ac4, NAME => 'TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:40:01,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42627 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f970bd9f1e861e2b64a2fb87fb209ac4, NAME => 'TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f970bd9f1e861e2b64a2fb87fb209ac4, NAME => 'TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:40:01,401 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:01,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49566 deadline: 1732246861399, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:01,401 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:01,401 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:01,402 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:40:01,401 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:01,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49590 deadline: 1732246861399, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:01,402 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:01,402 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:01,402 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:40:01,402 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:01,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49612 deadline: 1732246861399, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:01,403 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:01,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49582 deadline: 1732246861400, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:01,403 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:01,403 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:01,403 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:01,403 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:40:01,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49606 deadline: 1732246861400, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:01,403 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:01,403 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:01,403 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:40:01,403 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:01,403 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:01,403 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:40:01,525 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42545 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-22T03:40:01,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4. 2024-11-22T03:40:01,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4. as already flushing 2024-11-22T03:40:01,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4. 2024-11-22T03:40:01,525 ERROR [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => f970bd9f1e861e2b64a2fb87fb209ac4, NAME => 'TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:40:01,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => f970bd9f1e861e2b64a2fb87fb209ac4, NAME => 'TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:40:01,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42627 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f970bd9f1e861e2b64a2fb87fb209ac4, NAME => 'TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f970bd9f1e861e2b64a2fb87fb209ac4, NAME => 'TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:40:01,622 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=234 (bloomFilter=true), to=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/.tmp/B/78bf0c831baf4229a20ac284a6e5da48 2024-11-22T03:40:01,637 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/.tmp/C/5ff0a1cd4e424f10bffb4253faeb07d0 is 50, key is test_row_0/C:col10/1732246800075/Put/seqid=0 2024-11-22T03:40:01,651 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073741884_1060 (size=12151) 2024-11-22T03:40:01,652 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=234 (bloomFilter=true), to=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/.tmp/C/5ff0a1cd4e424f10bffb4253faeb07d0 2024-11-22T03:40:01,664 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/.tmp/A/ab5718b4e15340b08839c04e41e389eb as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/A/ab5718b4e15340b08839c04e41e389eb 2024-11-22T03:40:01,679 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/A/ab5718b4e15340b08839c04e41e389eb, entries=200, sequenceid=234, filesize=14.2 K 2024-11-22T03:40:01,679 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42545 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-22T03:40:01,679 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4. 2024-11-22T03:40:01,679 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4. as already flushing 2024-11-22T03:40:01,679 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4. 2024-11-22T03:40:01,680 ERROR [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => f970bd9f1e861e2b64a2fb87fb209ac4, NAME => 'TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:40:01,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => f970bd9f1e861e2b64a2fb87fb209ac4, NAME => 'TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:40:01,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42627 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f970bd9f1e861e2b64a2fb87fb209ac4, NAME => 'TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f970bd9f1e861e2b64a2fb87fb209ac4, NAME => 'TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:40:01,682 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/.tmp/B/78bf0c831baf4229a20ac284a6e5da48 as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/B/78bf0c831baf4229a20ac284a6e5da48 2024-11-22T03:40:01,700 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/B/78bf0c831baf4229a20ac284a6e5da48, entries=150, sequenceid=234, filesize=11.9 K 2024-11-22T03:40:01,701 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/.tmp/C/5ff0a1cd4e424f10bffb4253faeb07d0 as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/C/5ff0a1cd4e424f10bffb4253faeb07d0 2024-11-22T03:40:01,718 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/C/5ff0a1cd4e424f10bffb4253faeb07d0, entries=150, sequenceid=234, filesize=11.9 K 2024-11-22T03:40:01,720 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~299.53 KB/306720, currentSize=87.22 KB/89310 for f970bd9f1e861e2b64a2fb87fb209ac4 in 1000ms, sequenceid=234, compaction requested=true 2024-11-22T03:40:01,720 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f970bd9f1e861e2b64a2fb87fb209ac4: 2024-11-22T03:40:01,720 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f970bd9f1e861e2b64a2fb87fb209ac4:A, priority=-2147483648, current under compaction store size is 1 2024-11-22T03:40:01,720 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T03:40:01,720 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T03:40:01,721 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f970bd9f1e861e2b64a2fb87fb209ac4:B, priority=-2147483648, current under compaction store size is 2 2024-11-22T03:40:01,721 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T03:40:01,721 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T03:40:01,721 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f970bd9f1e861e2b64a2fb87fb209ac4:C, priority=-2147483648, current under compaction store size is 3 2024-11-22T03:40:01,722 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T03:40:01,722 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HStore(1541): f970bd9f1e861e2b64a2fb87fb209ac4/B is initiating minor compaction (all files) 2024-11-22T03:40:01,722 INFO [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f970bd9f1e861e2b64a2fb87fb209ac4/B in TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4. 2024-11-22T03:40:01,722 INFO [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/B/a694ac35e4c94035986bf5f8175a2f16, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/B/b5b38c5af776443aa690171ccc4f2d89, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/B/78bf0c831baf4229a20ac284a6e5da48] into tmpdir=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/.tmp, totalSize=36.0 K 2024-11-22T03:40:01,722 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T03:40:01,722 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 41677 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T03:40:01,723 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.HStore(1541): f970bd9f1e861e2b64a2fb87fb209ac4/A is initiating minor compaction (all files) 2024-11-22T03:40:01,723 INFO [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f970bd9f1e861e2b64a2fb87fb209ac4/A in TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4. 2024-11-22T03:40:01,723 INFO [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/A/00f669db6b0b4f248e23aea844f17710, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/A/79873c10df23470e91a8370dd7307ef0, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/A/ab5718b4e15340b08839c04e41e389eb] into tmpdir=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/.tmp, totalSize=40.7 K 2024-11-22T03:40:01,723 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] compactions.Compactor(225): Compacting a694ac35e4c94035986bf5f8175a2f16, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1732246799572 2024-11-22T03:40:01,723 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] compactions.Compactor(225): Compacting 00f669db6b0b4f248e23aea844f17710, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1732246799572 2024-11-22T03:40:01,724 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] compactions.Compactor(225): Compacting 79873c10df23470e91a8370dd7307ef0, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=212, earliestPutTs=1732246799718 2024-11-22T03:40:01,724 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] compactions.Compactor(225): Compacting b5b38c5af776443aa690171ccc4f2d89, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=212, earliestPutTs=1732246799723 2024-11-22T03:40:01,724 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] compactions.Compactor(225): Compacting ab5718b4e15340b08839c04e41e389eb, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=234, earliestPutTs=1732246800075 2024-11-22T03:40:01,724 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] compactions.Compactor(225): Compacting 78bf0c831baf4229a20ac284a6e5da48, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=234, earliestPutTs=1732246800075 2024-11-22T03:40:01,755 INFO [RS:0;c85114ed5096:42545-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f970bd9f1e861e2b64a2fb87fb209ac4#A#compaction#48 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-22T03:40:01,756 INFO [RS:0;c85114ed5096:42545-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f970bd9f1e861e2b64a2fb87fb209ac4#B#compaction#49 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T03:40:01,756 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/.tmp/A/c2cfd0728eb0486ebd70c72ce57458ac is 50, key is test_row_0/A:col10/1732246800075/Put/seqid=0 2024-11-22T03:40:01,756 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/.tmp/B/cb1e184b4be04ff88e908f63efa0280d is 50, key is test_row_0/B:col10/1732246800075/Put/seqid=0 2024-11-22T03:40:01,799 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073741885_1061 (size=12697) 2024-11-22T03:40:01,809 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/.tmp/A/c2cfd0728eb0486ebd70c72ce57458ac as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/A/c2cfd0728eb0486ebd70c72ce57458ac 2024-11-22T03:40:01,822 INFO [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f970bd9f1e861e2b64a2fb87fb209ac4/A of f970bd9f1e861e2b64a2fb87fb209ac4 into c2cfd0728eb0486ebd70c72ce57458ac(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T03:40:01,822 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f970bd9f1e861e2b64a2fb87fb209ac4: 2024-11-22T03:40:01,822 INFO [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., storeName=f970bd9f1e861e2b64a2fb87fb209ac4/A, priority=13, startTime=1732246801720; duration=0sec 2024-11-22T03:40:01,822 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T03:40:01,822 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f970bd9f1e861e2b64a2fb87fb209ac4:A 2024-11-22T03:40:01,823 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T03:40:01,825 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T03:40:01,826 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.HStore(1541): f970bd9f1e861e2b64a2fb87fb209ac4/C is initiating minor compaction (all files) 2024-11-22T03:40:01,826 INFO [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f970bd9f1e861e2b64a2fb87fb209ac4/C in TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4. 2024-11-22T03:40:01,826 INFO [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/C/60e84e1c86ad4cd3aaa3d684cc06a3ad, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/C/6fae8a2e9abd4328acd0050de197d16c, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/C/5ff0a1cd4e424f10bffb4253faeb07d0] into tmpdir=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/.tmp, totalSize=36.0 K 2024-11-22T03:40:01,827 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] compactions.Compactor(225): Compacting 60e84e1c86ad4cd3aaa3d684cc06a3ad, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1732246799572 2024-11-22T03:40:01,827 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6fae8a2e9abd4328acd0050de197d16c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=212, earliestPutTs=1732246799723 2024-11-22T03:40:01,828 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5ff0a1cd4e424f10bffb4253faeb07d0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=234, earliestPutTs=1732246800075 2024-11-22T03:40:01,834 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42545 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-22T03:40:01,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4. 2024-11-22T03:40:01,835 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2902): Flushing f970bd9f1e861e2b64a2fb87fb209ac4 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2024-11-22T03:40:01,839 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073741886_1062 (size=12697) 2024-11-22T03:40:01,844 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/.tmp/A/d89aa080a04b44c6b46fa211652eee8f is 50, key is test_row_0/A:col10/1732246800762/Put/seqid=0 2024-11-22T03:40:01,852 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/.tmp/B/cb1e184b4be04ff88e908f63efa0280d as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/B/cb1e184b4be04ff88e908f63efa0280d 2024-11-22T03:40:01,855 INFO [RS:0;c85114ed5096:42545-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f970bd9f1e861e2b64a2fb87fb209ac4#C#compaction#51 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T03:40:01,855 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/.tmp/C/790cd2d9acd542e09d2c8d1c49631c80 is 50, key is test_row_0/C:col10/1732246800075/Put/seqid=0 2024-11-22T03:40:01,862 INFO [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f970bd9f1e861e2b64a2fb87fb209ac4/B of f970bd9f1e861e2b64a2fb87fb209ac4 into cb1e184b4be04ff88e908f63efa0280d(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T03:40:01,862 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f970bd9f1e861e2b64a2fb87fb209ac4: 2024-11-22T03:40:01,862 INFO [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., storeName=f970bd9f1e861e2b64a2fb87fb209ac4/B, priority=13, startTime=1732246801720; duration=0sec 2024-11-22T03:40:01,863 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T03:40:01,863 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f970bd9f1e861e2b64a2fb87fb209ac4:B 2024-11-22T03:40:01,874 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073741887_1063 (size=12151) 2024-11-22T03:40:01,875 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=252 (bloomFilter=true), to=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/.tmp/A/d89aa080a04b44c6b46fa211652eee8f 2024-11-22T03:40:01,880 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073741888_1064 (size=12697) 2024-11-22T03:40:01,905 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/.tmp/C/790cd2d9acd542e09d2c8d1c49631c80 as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/C/790cd2d9acd542e09d2c8d1c49631c80 2024-11-22T03:40:01,909 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/.tmp/B/7aecc4f8d3c345f89aeaaf86b74860d1 is 50, key is test_row_0/B:col10/1732246800762/Put/seqid=0 2024-11-22T03:40:01,913 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4. as already flushing 2024-11-22T03:40:01,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(8855): Flush requested on f970bd9f1e861e2b64a2fb87fb209ac4 2024-11-22T03:40:01,920 INFO [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f970bd9f1e861e2b64a2fb87fb209ac4/C of f970bd9f1e861e2b64a2fb87fb209ac4 into 790cd2d9acd542e09d2c8d1c49631c80(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T03:40:01,920 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f970bd9f1e861e2b64a2fb87fb209ac4: 2024-11-22T03:40:01,920 INFO [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., storeName=f970bd9f1e861e2b64a2fb87fb209ac4/C, priority=13, startTime=1732246801721; duration=0sec 2024-11-22T03:40:01,920 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T03:40:01,920 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f970bd9f1e861e2b64a2fb87fb209ac4:C 2024-11-22T03:40:01,945 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073741889_1065 (size=12151) 2024-11-22T03:40:01,951 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:01,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49566 deadline: 1732246861944, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:01,952 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:01,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49582 deadline: 1732246861947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:01,953 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:01,953 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:01,953 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:40:01,953 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:01,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49612 deadline: 1732246861948, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:01,954 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:01,954 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:01,954 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:40:01,954 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:01,954 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:01,954 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:40:01,954 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:01,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49590 deadline: 1732246861949, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:01,955 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:01,955 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:01,955 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:01,955 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:40:01,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49606 deadline: 1732246861951, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:01,955 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:01,955 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:01,955 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:40:02,061 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:02,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49582 deadline: 1732246862059, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:02,062 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:02,062 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:02,062 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:02,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49566 deadline: 1732246862059, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:02,062 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:40:02,062 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:02,062 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:02,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49612 deadline: 1732246862059, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:02,062 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:02,062 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:40:02,062 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:02,063 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:02,063 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:02,063 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:40:02,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49590 deadline: 1732246862059, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:02,063 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:02,063 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:02,063 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:02,063 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:40:02,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49606 deadline: 1732246862060, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:02,063 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:02,064 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:02,064 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:40:02,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-22T03:40:02,270 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:02,270 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:02,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49582 deadline: 1732246862269, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:02,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49566 deadline: 1732246862269, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:02,271 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:02,271 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:02,271 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:02,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49606 deadline: 1732246862270, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:02,271 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:02,271 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:02,271 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:40:02,271 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:40:02,272 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:02,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49612 deadline: 1732246862270, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:02,272 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:02,272 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:02,272 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:40:02,272 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:02,272 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:02,272 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:40:02,272 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:02,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49590 deadline: 1732246862270, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:02,273 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:02,273 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:02,273 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:40:02,346 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=252 (bloomFilter=true), to=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/.tmp/B/7aecc4f8d3c345f89aeaaf86b74860d1 2024-11-22T03:40:02,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/.tmp/C/4a506faf14c64ba0be946fea194981f9 is 50, key is test_row_0/C:col10/1732246800762/Put/seqid=0 2024-11-22T03:40:02,370 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073741890_1066 (size=12151) 2024-11-22T03:40:02,370 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=252 (bloomFilter=true), to=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/.tmp/C/4a506faf14c64ba0be946fea194981f9 2024-11-22T03:40:02,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/.tmp/A/d89aa080a04b44c6b46fa211652eee8f as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/A/d89aa080a04b44c6b46fa211652eee8f 2024-11-22T03:40:02,399 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/A/d89aa080a04b44c6b46fa211652eee8f, entries=150, sequenceid=252, filesize=11.9 K 2024-11-22T03:40:02,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/.tmp/B/7aecc4f8d3c345f89aeaaf86b74860d1 as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/B/7aecc4f8d3c345f89aeaaf86b74860d1 2024-11-22T03:40:02,408 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/B/7aecc4f8d3c345f89aeaaf86b74860d1, entries=150, sequenceid=252, filesize=11.9 K 2024-11-22T03:40:02,409 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/.tmp/C/4a506faf14c64ba0be946fea194981f9 as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/C/4a506faf14c64ba0be946fea194981f9 2024-11-22T03:40:02,422 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/C/4a506faf14c64ba0be946fea194981f9, entries=150, sequenceid=252, filesize=11.9 K 2024-11-22T03:40:02,423 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=114.05 KB/116790 for f970bd9f1e861e2b64a2fb87fb209ac4 in 589ms, sequenceid=252, compaction requested=false 2024-11-22T03:40:02,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2603): Flush status journal for f970bd9f1e861e2b64a2fb87fb209ac4: 2024-11-22T03:40:02,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4. 2024-11-22T03:40:02,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=18 2024-11-22T03:40:02,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42627 {}] master.HMaster(4169): Remote procedure done, pid=18 2024-11-22T03:40:02,427 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=18, resume processing ppid=17 2024-11-22T03:40:02,428 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.4570 sec 2024-11-22T03:40:02,431 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees in 2.4660 sec 2024-11-22T03:40:02,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(8855): Flush requested on f970bd9f1e861e2b64a2fb87fb209ac4 2024-11-22T03:40:02,581 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-22T03:40:02,581 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f970bd9f1e861e2b64a2fb87fb209ac4 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2024-11-22T03:40:02,601 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/.tmp/A/a0cec3316e6c4395ab8f80f15ed72fee is 50, key is test_row_0/A:col10/1732246801949/Put/seqid=0 2024-11-22T03:40:02,602 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:02,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49566 deadline: 1732246862598, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:02,603 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:02,604 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:02,604 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:02,604 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:40:02,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49590 deadline: 1732246862600, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:02,604 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:02,604 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:02,604 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:40:02,606 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:02,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49606 deadline: 1732246862603, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:02,607 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:02,607 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:02,607 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:40:02,607 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:02,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49612 deadline: 1732246862603, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:02,607 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:02,608 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:02,608 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:40:02,608 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:02,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49582 deadline: 1732246862603, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:02,608 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:02,608 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:02,608 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:40:02,619 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073741891_1067 (size=14741) 2024-11-22T03:40:02,710 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:02,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49566 deadline: 1732246862709, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:02,711 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:02,711 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:02,711 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:40:02,712 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:02,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49590 deadline: 1732246862709, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:02,713 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:02,713 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:02,713 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:40:02,721 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:02,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49606 deadline: 1732246862718, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:02,721 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:02,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49612 deadline: 1732246862719, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:02,722 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:02,722 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:02,722 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:02,722 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:40:02,722 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:02,722 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:02,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49582 deadline: 1732246862719, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:02,722 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:40:02,723 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:02,723 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:02,723 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:40:02,919 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:02,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49566 deadline: 1732246862918, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:02,920 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:02,920 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:02,920 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:02,920 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:40:02,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49590 deadline: 1732246862919, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:02,920 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:02,920 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:02,920 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:40:02,929 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:02,929 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:02,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49612 deadline: 1732246862929, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:02,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49606 deadline: 1732246862929, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:02,930 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:02,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49582 deadline: 1732246862929, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:02,930 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:02,930 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:02,931 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:02,930 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:02,931 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:40:02,931 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:02,931 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:02,931 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:40:02,931 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:40:03,020 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=275 (bloomFilter=true), to=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/.tmp/A/a0cec3316e6c4395ab8f80f15ed72fee 2024-11-22T03:40:03,032 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/.tmp/B/7cfe5175ae7649eda140430a80603695 is 50, key is test_row_0/B:col10/1732246801949/Put/seqid=0 2024-11-22T03:40:03,036 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073741892_1068 (size=12301) 2024-11-22T03:40:03,229 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:03,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49566 deadline: 1732246863228, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:03,230 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:03,230 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:03,230 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:03,230 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:40:03,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49590 deadline: 1732246863229, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:03,230 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:03,230 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:03,230 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:40:03,241 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:03,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49612 deadline: 1732246863239, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:03,241 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:03,242 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:03,242 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:03,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49606 deadline: 1732246863239, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:03,242 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:40:03,242 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:03,242 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:03,242 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:03,242 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:40:03,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49582 deadline: 1732246863239, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:03,242 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:03,242 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:03,242 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:40:03,437 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=275 (bloomFilter=true), to=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/.tmp/B/7cfe5175ae7649eda140430a80603695 2024-11-22T03:40:03,448 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/.tmp/C/477966aea94140e98f95e2921a241b01 is 50, key is test_row_0/C:col10/1732246801949/Put/seqid=0 2024-11-22T03:40:03,464 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073741893_1069 (size=12301) 2024-11-22T03:40:03,739 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:03,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49566 deadline: 1732246863738, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:03,739 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:03,740 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:03,740 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:40:03,740 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:03,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49590 deadline: 1732246863739, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:03,740 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:03,741 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:03,741 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:40:03,749 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:03,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49612 deadline: 1732246863749, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:03,750 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:03,750 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:03,750 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:03,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49606 deadline: 1732246863749, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:03,750 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:40:03,750 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:03,751 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:03,751 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:40:03,751 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:03,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49582 deadline: 1732246863749, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:03,751 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:03,751 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:03,751 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:40:03,866 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=275 (bloomFilter=true), to=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/.tmp/C/477966aea94140e98f95e2921a241b01 2024-11-22T03:40:03,876 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/.tmp/A/a0cec3316e6c4395ab8f80f15ed72fee as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/A/a0cec3316e6c4395ab8f80f15ed72fee 2024-11-22T03:40:03,882 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/A/a0cec3316e6c4395ab8f80f15ed72fee, entries=200, sequenceid=275, filesize=14.4 K 2024-11-22T03:40:03,883 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/.tmp/B/7cfe5175ae7649eda140430a80603695 as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/B/7cfe5175ae7649eda140430a80603695 2024-11-22T03:40:03,889 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/B/7cfe5175ae7649eda140430a80603695, entries=150, sequenceid=275, filesize=12.0 K 2024-11-22T03:40:03,892 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/.tmp/C/477966aea94140e98f95e2921a241b01 as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/C/477966aea94140e98f95e2921a241b01 2024-11-22T03:40:03,902 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/C/477966aea94140e98f95e2921a241b01, entries=150, sequenceid=275, filesize=12.0 K 2024-11-22T03:40:03,904 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~317.11 KB/324720, currentSize=93.93 KB/96180 for f970bd9f1e861e2b64a2fb87fb209ac4 in 1323ms, sequenceid=275, compaction requested=true 2024-11-22T03:40:03,904 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f970bd9f1e861e2b64a2fb87fb209ac4: 2024-11-22T03:40:03,904 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T03:40:03,904 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f970bd9f1e861e2b64a2fb87fb209ac4:A, priority=-2147483648, current under compaction store size is 1 2024-11-22T03:40:03,905 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T03:40:03,905 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f970bd9f1e861e2b64a2fb87fb209ac4:B, priority=-2147483648, current under compaction store size is 2 2024-11-22T03:40:03,905 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T03:40:03,905 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T03:40:03,905 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f970bd9f1e861e2b64a2fb87fb209ac4:C, priority=-2147483648, current under compaction store size is 3 2024-11-22T03:40:03,905 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T03:40:03,908 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39589 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T03:40:03,908 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.HStore(1541): f970bd9f1e861e2b64a2fb87fb209ac4/A is initiating minor compaction (all files) 2024-11-22T03:40:03,908 INFO [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f970bd9f1e861e2b64a2fb87fb209ac4/A in TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4. 2024-11-22T03:40:03,908 INFO [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/A/c2cfd0728eb0486ebd70c72ce57458ac, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/A/d89aa080a04b44c6b46fa211652eee8f, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/A/a0cec3316e6c4395ab8f80f15ed72fee] into tmpdir=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/.tmp, totalSize=38.7 K 2024-11-22T03:40:03,908 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37149 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T03:40:03,909 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HStore(1541): f970bd9f1e861e2b64a2fb87fb209ac4/B is initiating minor compaction (all files) 2024-11-22T03:40:03,909 INFO [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f970bd9f1e861e2b64a2fb87fb209ac4/B in TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4. 2024-11-22T03:40:03,909 INFO [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/B/cb1e184b4be04ff88e908f63efa0280d, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/B/7aecc4f8d3c345f89aeaaf86b74860d1, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/B/7cfe5175ae7649eda140430a80603695] into tmpdir=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/.tmp, totalSize=36.3 K 2024-11-22T03:40:03,911 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] compactions.Compactor(225): Compacting cb1e184b4be04ff88e908f63efa0280d, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=234, earliestPutTs=1732246800075 2024-11-22T03:40:03,911 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] compactions.Compactor(225): Compacting c2cfd0728eb0486ebd70c72ce57458ac, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=234, earliestPutTs=1732246800075 2024-11-22T03:40:03,912 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] compactions.Compactor(225): Compacting d89aa080a04b44c6b46fa211652eee8f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=252, earliestPutTs=1732246800739 2024-11-22T03:40:03,912 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] compactions.Compactor(225): Compacting 7aecc4f8d3c345f89aeaaf86b74860d1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=252, earliestPutTs=1732246800739 2024-11-22T03:40:03,913 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] compactions.Compactor(225): Compacting a0cec3316e6c4395ab8f80f15ed72fee, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=275, earliestPutTs=1732246801914 2024-11-22T03:40:03,913 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] compactions.Compactor(225): Compacting 7cfe5175ae7649eda140430a80603695, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=275, earliestPutTs=1732246801914 2024-11-22T03:40:03,936 INFO [RS:0;c85114ed5096:42545-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f970bd9f1e861e2b64a2fb87fb209ac4#B#compaction#57 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T03:40:03,937 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/.tmp/B/88db1860aa1b443c92cfdfd6a9e386a8 is 50, key is test_row_0/B:col10/1732246801949/Put/seqid=0 2024-11-22T03:40:03,939 INFO [RS:0;c85114ed5096:42545-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f970bd9f1e861e2b64a2fb87fb209ac4#A#compaction#58 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T03:40:03,940 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/.tmp/A/32714810e79943e489b8965490a9deb0 is 50, key is test_row_0/A:col10/1732246801949/Put/seqid=0 2024-11-22T03:40:03,958 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073741894_1070 (size=12949) 2024-11-22T03:40:03,967 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/.tmp/B/88db1860aa1b443c92cfdfd6a9e386a8 as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/B/88db1860aa1b443c92cfdfd6a9e386a8 2024-11-22T03:40:03,977 INFO [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f970bd9f1e861e2b64a2fb87fb209ac4/B of f970bd9f1e861e2b64a2fb87fb209ac4 into 88db1860aa1b443c92cfdfd6a9e386a8(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T03:40:03,977 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f970bd9f1e861e2b64a2fb87fb209ac4: 2024-11-22T03:40:03,977 INFO [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., storeName=f970bd9f1e861e2b64a2fb87fb209ac4/B, priority=13, startTime=1732246803905; duration=0sec 2024-11-22T03:40:03,977 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T03:40:03,977 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f970bd9f1e861e2b64a2fb87fb209ac4:B 2024-11-22T03:40:03,978 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T03:40:03,980 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37149 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T03:40:03,980 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HStore(1541): f970bd9f1e861e2b64a2fb87fb209ac4/C is initiating minor compaction (all files) 2024-11-22T03:40:03,980 INFO [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f970bd9f1e861e2b64a2fb87fb209ac4/C in TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4. 2024-11-22T03:40:03,981 INFO [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/C/790cd2d9acd542e09d2c8d1c49631c80, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/C/4a506faf14c64ba0be946fea194981f9, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/C/477966aea94140e98f95e2921a241b01] into tmpdir=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/.tmp, totalSize=36.3 K 2024-11-22T03:40:03,981 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] compactions.Compactor(225): Compacting 790cd2d9acd542e09d2c8d1c49631c80, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=234, earliestPutTs=1732246800075 2024-11-22T03:40:03,982 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] compactions.Compactor(225): Compacting 4a506faf14c64ba0be946fea194981f9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=252, earliestPutTs=1732246800739 2024-11-22T03:40:03,982 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] compactions.Compactor(225): Compacting 477966aea94140e98f95e2921a241b01, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=275, earliestPutTs=1732246801914 2024-11-22T03:40:04,000 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073741895_1071 (size=12949) 2024-11-22T03:40:04,011 INFO [RS:0;c85114ed5096:42545-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f970bd9f1e861e2b64a2fb87fb209ac4#C#compaction#59 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T03:40:04,014 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/.tmp/C/9502f711563044d5bcdc7557806f0d89 is 50, key is test_row_0/C:col10/1732246801949/Put/seqid=0 2024-11-22T03:40:04,028 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073741896_1072 (size=12949) 2024-11-22T03:40:04,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-22T03:40:04,119 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-22T03:40:04,119 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42627 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-22T03:40:04,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42627 {}] procedure2.ProcedureExecutor(1139): Stored pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees 2024-11-22T03:40:04,123 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-22T03:40:04,124 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-22T03:40:04,124 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=20, ppid=19, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-22T03:40:04,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-22T03:40:04,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-22T03:40:04,278 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42545 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-22T03:40:04,278 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4. 2024-11-22T03:40:04,279 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2902): Flushing f970bd9f1e861e2b64a2fb87fb209ac4 3/3 column families, dataSize=93.93 KB heapSize=246.84 KB 2024-11-22T03:40:04,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/.tmp/A/2a450a145c3044f69195ccd86386ed94 is 50, key is test_row_0/A:col10/1732246802590/Put/seqid=0 2024-11-22T03:40:04,309 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073741897_1073 (size=12301) 2024-11-22T03:40:04,314 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=293 (bloomFilter=true), to=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/.tmp/A/2a450a145c3044f69195ccd86386ed94 2024-11-22T03:40:04,330 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/.tmp/B/521ceb8dc886461da9ae9e46b5b62b8f is 50, key is test_row_0/B:col10/1732246802590/Put/seqid=0 2024-11-22T03:40:04,340 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073741898_1074 (size=12301) 2024-11-22T03:40:04,343 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=293 (bloomFilter=true), to=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/.tmp/B/521ceb8dc886461da9ae9e46b5b62b8f 2024-11-22T03:40:04,369 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/.tmp/C/5a207447ec7d4403b1520d0267c51f62 is 50, key is test_row_0/C:col10/1732246802590/Put/seqid=0 2024-11-22T03:40:04,389 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073741899_1075 (size=12301) 2024-11-22T03:40:04,411 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/.tmp/A/32714810e79943e489b8965490a9deb0 as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/A/32714810e79943e489b8965490a9deb0 2024-11-22T03:40:04,421 INFO [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f970bd9f1e861e2b64a2fb87fb209ac4/A of f970bd9f1e861e2b64a2fb87fb209ac4 into 32714810e79943e489b8965490a9deb0(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T03:40:04,421 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f970bd9f1e861e2b64a2fb87fb209ac4: 2024-11-22T03:40:04,421 INFO [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., storeName=f970bd9f1e861e2b64a2fb87fb209ac4/A, priority=13, startTime=1732246803904; duration=0sec 2024-11-22T03:40:04,421 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T03:40:04,421 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f970bd9f1e861e2b64a2fb87fb209ac4:A 2024-11-22T03:40:04,442 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/.tmp/C/9502f711563044d5bcdc7557806f0d89 as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/C/9502f711563044d5bcdc7557806f0d89 2024-11-22T03:40:04,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-22T03:40:04,455 INFO [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f970bd9f1e861e2b64a2fb87fb209ac4/C of f970bd9f1e861e2b64a2fb87fb209ac4 into 9502f711563044d5bcdc7557806f0d89(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T03:40:04,455 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f970bd9f1e861e2b64a2fb87fb209ac4: 2024-11-22T03:40:04,455 INFO [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., storeName=f970bd9f1e861e2b64a2fb87fb209ac4/C, priority=13, startTime=1732246803905; duration=0sec 2024-11-22T03:40:04,455 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T03:40:04,455 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f970bd9f1e861e2b64a2fb87fb209ac4:C 2024-11-22T03:40:04,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-22T03:40:04,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(8855): Flush requested on f970bd9f1e861e2b64a2fb87fb209ac4 2024-11-22T03:40:04,762 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4. as already flushing 2024-11-22T03:40:04,791 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:04,791 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=293 (bloomFilter=true), to=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/.tmp/C/5a207447ec7d4403b1520d0267c51f62 2024-11-22T03:40:04,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49590 deadline: 1732246864782, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:04,792 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:04,792 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:04,792 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:04,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49582 deadline: 1732246864787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:04,792 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:04,792 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:40:04,793 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:04,793 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:04,793 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:40:04,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49612 deadline: 1732246864787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:04,794 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:04,794 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:04,794 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:40:04,794 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:04,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49566 deadline: 1732246864788, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:04,795 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:04,795 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:04,795 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:40:04,801 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:04,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49606 deadline: 1732246864797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:04,803 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:04,803 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:04,803 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:40:04,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/.tmp/A/2a450a145c3044f69195ccd86386ed94 as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/A/2a450a145c3044f69195ccd86386ed94 2024-11-22T03:40:04,814 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/A/2a450a145c3044f69195ccd86386ed94, entries=150, sequenceid=293, filesize=12.0 K 2024-11-22T03:40:04,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/.tmp/B/521ceb8dc886461da9ae9e46b5b62b8f as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/B/521ceb8dc886461da9ae9e46b5b62b8f 2024-11-22T03:40:04,826 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/B/521ceb8dc886461da9ae9e46b5b62b8f, entries=150, sequenceid=293, filesize=12.0 K 2024-11-22T03:40:04,828 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/.tmp/C/5a207447ec7d4403b1520d0267c51f62 as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/C/5a207447ec7d4403b1520d0267c51f62 2024-11-22T03:40:04,839 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/C/5a207447ec7d4403b1520d0267c51f62, entries=150, sequenceid=293, filesize=12.0 K 2024-11-22T03:40:04,840 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~246.80 KB/252720, currentSize=107.34 KB/109920 for f970bd9f1e861e2b64a2fb87fb209ac4 in 562ms, sequenceid=293, compaction requested=false 2024-11-22T03:40:04,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2603): Flush status journal for f970bd9f1e861e2b64a2fb87fb209ac4: 2024-11-22T03:40:04,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4. 2024-11-22T03:40:04,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=20 2024-11-22T03:40:04,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42627 {}] master.HMaster(4169): Remote procedure done, pid=20 2024-11-22T03:40:04,848 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=20, resume processing ppid=19 2024-11-22T03:40:04,848 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, ppid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 721 msec 2024-11-22T03:40:04,853 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees in 729 msec 2024-11-22T03:40:04,901 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-22T03:40:04,901 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f970bd9f1e861e2b64a2fb87fb209ac4 3/3 column families, dataSize=114.05 KB heapSize=299.58 KB 2024-11-22T03:40:04,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(8855): Flush requested on f970bd9f1e861e2b64a2fb87fb209ac4 2024-11-22T03:40:04,910 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/.tmp/A/daebc7c44ab644e79e0306a701ccae2c is 50, key is test_row_0/A:col10/1732246804785/Put/seqid=0 2024-11-22T03:40:04,929 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073741900_1076 (size=14741) 2024-11-22T03:40:04,931 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:04,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49582 deadline: 1732246864921, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:04,932 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:04,932 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:04,932 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:04,932 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:40:04,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49612 deadline: 1732246864923, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:04,933 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:04,933 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:04,933 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:40:04,933 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:04,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49590 deadline: 1732246864926, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:04,933 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:04,933 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:04,933 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:40:04,933 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:04,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49566 deadline: 1732246864929, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:04,936 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:04,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49606 deadline: 1732246864931, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:04,936 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:04,936 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:04,936 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:04,936 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:04,936 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:40:04,936 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:40:04,939 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=317 (bloomFilter=true), to=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/.tmp/A/daebc7c44ab644e79e0306a701ccae2c 2024-11-22T03:40:04,960 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/.tmp/B/c560f3405cd74dbeb62d9e4b37c3adde is 50, key is test_row_0/B:col10/1732246804785/Put/seqid=0 2024-11-22T03:40:05,001 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073741901_1077 (size=12301) 2024-11-22T03:40:05,002 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=317 (bloomFilter=true), to=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/.tmp/B/c560f3405cd74dbeb62d9e4b37c3adde 2024-11-22T03:40:05,021 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/.tmp/C/081b196b6c2c4cacaf395699683f5b84 is 50, key is test_row_0/C:col10/1732246804785/Put/seqid=0 2024-11-22T03:40:05,041 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:05,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49582 deadline: 1732246865039, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:05,041 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:05,041 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:05,042 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:05,042 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:40:05,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49612 deadline: 1732246865040, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:05,042 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:05,042 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:05,042 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:05,042 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:40:05,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49566 deadline: 1732246865040, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:05,043 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:05,043 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:05,043 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:40:05,043 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:05,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49606 deadline: 1732246865040, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:05,043 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:05,043 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:05,043 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:40:05,048 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:05,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49590 deadline: 1732246865040, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:05,049 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:05,049 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:05,049 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:40:05,054 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073741902_1078 (size=12301) 2024-11-22T03:40:05,055 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=317 (bloomFilter=true), to=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/.tmp/C/081b196b6c2c4cacaf395699683f5b84 2024-11-22T03:40:05,066 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/.tmp/A/daebc7c44ab644e79e0306a701ccae2c as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/A/daebc7c44ab644e79e0306a701ccae2c 2024-11-22T03:40:05,074 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/A/daebc7c44ab644e79e0306a701ccae2c, entries=200, sequenceid=317, filesize=14.4 K 2024-11-22T03:40:05,076 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/.tmp/B/c560f3405cd74dbeb62d9e4b37c3adde as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/B/c560f3405cd74dbeb62d9e4b37c3adde 2024-11-22T03:40:05,091 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/B/c560f3405cd74dbeb62d9e4b37c3adde, entries=150, sequenceid=317, filesize=12.0 K 2024-11-22T03:40:05,092 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/.tmp/C/081b196b6c2c4cacaf395699683f5b84 as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/C/081b196b6c2c4cacaf395699683f5b84 2024-11-22T03:40:05,101 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/C/081b196b6c2c4cacaf395699683f5b84, entries=150, sequenceid=317, filesize=12.0 K 2024-11-22T03:40:05,102 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=73.80 KB/75570 for f970bd9f1e861e2b64a2fb87fb209ac4 in 201ms, sequenceid=317, compaction requested=true 2024-11-22T03:40:05,102 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f970bd9f1e861e2b64a2fb87fb209ac4: 2024-11-22T03:40:05,102 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T03:40:05,103 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f970bd9f1e861e2b64a2fb87fb209ac4:A, priority=-2147483648, current under compaction store size is 1 2024-11-22T03:40:05,103 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T03:40:05,103 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T03:40:05,104 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f970bd9f1e861e2b64a2fb87fb209ac4:B, priority=-2147483648, current under compaction store size is 2 2024-11-22T03:40:05,104 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T03:40:05,104 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f970bd9f1e861e2b64a2fb87fb209ac4:C, priority=-2147483648, current under compaction store size is 3 2024-11-22T03:40:05,104 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T03:40:05,105 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39991 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T03:40:05,106 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.HStore(1541): f970bd9f1e861e2b64a2fb87fb209ac4/A is initiating minor compaction (all files) 2024-11-22T03:40:05,106 INFO [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f970bd9f1e861e2b64a2fb87fb209ac4/A in TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4. 2024-11-22T03:40:05,106 INFO [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/A/32714810e79943e489b8965490a9deb0, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/A/2a450a145c3044f69195ccd86386ed94, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/A/daebc7c44ab644e79e0306a701ccae2c] into tmpdir=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/.tmp, totalSize=39.1 K 2024-11-22T03:40:05,106 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] compactions.Compactor(225): Compacting 32714810e79943e489b8965490a9deb0, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=275, earliestPutTs=1732246801914 2024-11-22T03:40:05,106 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T03:40:05,106 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HStore(1541): f970bd9f1e861e2b64a2fb87fb209ac4/B is initiating minor compaction (all files) 2024-11-22T03:40:05,107 INFO [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f970bd9f1e861e2b64a2fb87fb209ac4/B in TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4. 2024-11-22T03:40:05,107 INFO [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/B/88db1860aa1b443c92cfdfd6a9e386a8, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/B/521ceb8dc886461da9ae9e46b5b62b8f, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/B/c560f3405cd74dbeb62d9e4b37c3adde] into tmpdir=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/.tmp, totalSize=36.7 K 2024-11-22T03:40:05,107 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] compactions.Compactor(225): Compacting 88db1860aa1b443c92cfdfd6a9e386a8, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=275, earliestPutTs=1732246801914 2024-11-22T03:40:05,107 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2a450a145c3044f69195ccd86386ed94, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=293, earliestPutTs=1732246802590 2024-11-22T03:40:05,108 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] compactions.Compactor(225): Compacting 521ceb8dc886461da9ae9e46b5b62b8f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=293, earliestPutTs=1732246802590 2024-11-22T03:40:05,108 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] compactions.Compactor(225): Compacting daebc7c44ab644e79e0306a701ccae2c, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=317, earliestPutTs=1732246804785 2024-11-22T03:40:05,108 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] compactions.Compactor(225): Compacting c560f3405cd74dbeb62d9e4b37c3adde, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=317, earliestPutTs=1732246804785 2024-11-22T03:40:05,119 INFO [RS:0;c85114ed5096:42545-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f970bd9f1e861e2b64a2fb87fb209ac4#A#compaction#66 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-22T03:40:05,119 INFO [RS:0;c85114ed5096:42545-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f970bd9f1e861e2b64a2fb87fb209ac4#B#compaction#67 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T03:40:05,119 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/.tmp/A/d405092cdcbd408caf5f8951e21805c3 is 50, key is test_row_0/A:col10/1732246804785/Put/seqid=0 2024-11-22T03:40:05,120 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/.tmp/B/7e919651fda74c34b20c4a3f74ef0026 is 50, key is test_row_0/B:col10/1732246804785/Put/seqid=0 2024-11-22T03:40:05,135 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073741904_1080 (size=13051) 2024-11-22T03:40:05,139 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073741903_1079 (size=13051) 2024-11-22T03:40:05,150 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/.tmp/A/d405092cdcbd408caf5f8951e21805c3 as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/A/d405092cdcbd408caf5f8951e21805c3 2024-11-22T03:40:05,157 INFO [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f970bd9f1e861e2b64a2fb87fb209ac4/A of f970bd9f1e861e2b64a2fb87fb209ac4 into d405092cdcbd408caf5f8951e21805c3(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T03:40:05,157 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f970bd9f1e861e2b64a2fb87fb209ac4: 2024-11-22T03:40:05,157 INFO [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., storeName=f970bd9f1e861e2b64a2fb87fb209ac4/A, priority=13, startTime=1732246805102; duration=0sec 2024-11-22T03:40:05,158 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T03:40:05,158 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f970bd9f1e861e2b64a2fb87fb209ac4:A 2024-11-22T03:40:05,158 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T03:40:05,161 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T03:40:05,161 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.HStore(1541): f970bd9f1e861e2b64a2fb87fb209ac4/C is initiating minor compaction (all files) 2024-11-22T03:40:05,161 INFO [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f970bd9f1e861e2b64a2fb87fb209ac4/C in TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4. 2024-11-22T03:40:05,162 INFO [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/C/9502f711563044d5bcdc7557806f0d89, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/C/5a207447ec7d4403b1520d0267c51f62, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/C/081b196b6c2c4cacaf395699683f5b84] into tmpdir=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/.tmp, totalSize=36.7 K 2024-11-22T03:40:05,162 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9502f711563044d5bcdc7557806f0d89, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=275, earliestPutTs=1732246801914 2024-11-22T03:40:05,164 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5a207447ec7d4403b1520d0267c51f62, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=293, earliestPutTs=1732246802590 2024-11-22T03:40:05,164 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] compactions.Compactor(225): Compacting 081b196b6c2c4cacaf395699683f5b84, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=317, earliestPutTs=1732246804785 2024-11-22T03:40:05,174 INFO [RS:0;c85114ed5096:42545-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f970bd9f1e861e2b64a2fb87fb209ac4#C#compaction#68 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T03:40:05,174 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/.tmp/C/3b798870b3c44845916d55d82f80abb4 is 50, key is test_row_0/C:col10/1732246804785/Put/seqid=0 2024-11-22T03:40:05,187 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073741905_1081 (size=13051) 2024-11-22T03:40:05,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(8855): Flush requested on f970bd9f1e861e2b64a2fb87fb209ac4 2024-11-22T03:40:05,251 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-22T03:40:05,252 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f970bd9f1e861e2b64a2fb87fb209ac4 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-22T03:40:05,259 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/.tmp/A/d6f08aef05934343b1f738a5442203b6 is 50, key is test_row_0/A:col10/1732246804925/Put/seqid=0 2024-11-22T03:40:05,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-22T03:40:05,268 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-22T03:40:05,269 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42627 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-22T03:40:05,270 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073741906_1082 (size=14741) 2024-11-22T03:40:05,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42627 {}] procedure2.ProcedureExecutor(1139): Stored pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees 2024-11-22T03:40:05,271 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=333 (bloomFilter=true), to=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/.tmp/A/d6f08aef05934343b1f738a5442203b6 2024-11-22T03:40:05,274 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-22T03:40:05,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-22T03:40:05,276 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-22T03:40:05,277 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=22, ppid=21, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-22T03:40:05,288 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:05,288 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/.tmp/B/549bca3e86484bba81f9a9d7402f3d35 is 50, key is test_row_0/B:col10/1732246804925/Put/seqid=0 2024-11-22T03:40:05,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49612 deadline: 1732246865282, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:05,288 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:05,288 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:05,288 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:05,288 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:40:05,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49566 deadline: 1732246865284, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:05,289 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:05,289 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:05,289 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:40:05,292 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:05,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49582 deadline: 1732246865286, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:05,293 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073741907_1083 (size=12301) 2024-11-22T03:40:05,293 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:05,293 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:05,293 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:40:05,293 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:05,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49590 deadline: 1732246865288, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:05,294 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:05,294 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:05,294 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:05,294 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:40:05,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49606 deadline: 1732246865288, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:05,294 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:05,294 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:05,295 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:40:05,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-22T03:40:05,399 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:05,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49566 deadline: 1732246865399, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:05,400 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:05,400 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:05,400 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:05,400 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:40:05,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49582 deadline: 1732246865399, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:05,400 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:05,400 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:05,400 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:40:05,400 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:05,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49612 deadline: 1732246865399, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:05,401 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:05,401 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:05,401 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:40:05,401 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:05,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49590 deadline: 1732246865399, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:05,401 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:05,401 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:05,401 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:05,401 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:40:05,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49606 deadline: 1732246865399, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:05,402 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:05,402 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:05,402 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:40:05,431 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42545 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-22T03:40:05,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4. 2024-11-22T03:40:05,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4. as already flushing 2024-11-22T03:40:05,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4. 2024-11-22T03:40:05,432 ERROR [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => f970bd9f1e861e2b64a2fb87fb209ac4, NAME => 'TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:40:05,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => f970bd9f1e861e2b64a2fb87fb209ac4, NAME => 'TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:40:05,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42627 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f970bd9f1e861e2b64a2fb87fb209ac4, NAME => 'TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f970bd9f1e861e2b64a2fb87fb209ac4, NAME => 'TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:40:05,554 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/.tmp/B/7e919651fda74c34b20c4a3f74ef0026 as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/B/7e919651fda74c34b20c4a3f74ef0026 2024-11-22T03:40:05,571 INFO [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f970bd9f1e861e2b64a2fb87fb209ac4/B of f970bd9f1e861e2b64a2fb87fb209ac4 into 7e919651fda74c34b20c4a3f74ef0026(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T03:40:05,571 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f970bd9f1e861e2b64a2fb87fb209ac4: 2024-11-22T03:40:05,571 INFO [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., storeName=f970bd9f1e861e2b64a2fb87fb209ac4/B, priority=13, startTime=1732246805103; duration=0sec 2024-11-22T03:40:05,571 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T03:40:05,571 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f970bd9f1e861e2b64a2fb87fb209ac4:B 2024-11-22T03:40:05,586 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42545 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-22T03:40:05,586 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4. 2024-11-22T03:40:05,586 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4. as already flushing 2024-11-22T03:40:05,586 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4. 2024-11-22T03:40:05,586 ERROR [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => f970bd9f1e861e2b64a2fb87fb209ac4, NAME => 'TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:40:05,586 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => f970bd9f1e861e2b64a2fb87fb209ac4, NAME => 'TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:40:05,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42627 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f970bd9f1e861e2b64a2fb87fb209ac4, NAME => 'TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f970bd9f1e861e2b64a2fb87fb209ac4, NAME => 'TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:40:05,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-22T03:40:05,595 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/.tmp/C/3b798870b3c44845916d55d82f80abb4 as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/C/3b798870b3c44845916d55d82f80abb4 2024-11-22T03:40:05,612 INFO [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f970bd9f1e861e2b64a2fb87fb209ac4/C of f970bd9f1e861e2b64a2fb87fb209ac4 into 3b798870b3c44845916d55d82f80abb4(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T03:40:05,612 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:05,612 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f970bd9f1e861e2b64a2fb87fb209ac4: 2024-11-22T03:40:05,612 INFO [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., storeName=f970bd9f1e861e2b64a2fb87fb209ac4/C, priority=13, startTime=1732246805104; duration=0sec 2024-11-22T03:40:05,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49566 deadline: 1732246865608, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:05,612 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T03:40:05,613 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f970bd9f1e861e2b64a2fb87fb209ac4:C 2024-11-22T03:40:05,613 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:05,613 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:05,613 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:40:05,613 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:05,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49590 deadline: 1732246865609, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:05,614 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:05,614 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:05,614 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:40:05,614 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:05,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49612 deadline: 1732246865609, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:05,614 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:05,614 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:05,615 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:40:05,614 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:05,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49582 deadline: 1732246865609, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:05,615 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:05,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49606 deadline: 1732246865609, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:05,615 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:05,615 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:05,615 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:40:05,615 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:05,615 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:05,615 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:40:05,695 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=333 (bloomFilter=true), to=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/.tmp/B/549bca3e86484bba81f9a9d7402f3d35 2024-11-22T03:40:05,711 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/.tmp/C/e15a26d1669c4c1baa758db1a82ca2a3 is 50, key is test_row_0/C:col10/1732246804925/Put/seqid=0 2024-11-22T03:40:05,726 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073741908_1084 (size=12301) 2024-11-22T03:40:05,727 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=333 (bloomFilter=true), to=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/.tmp/C/e15a26d1669c4c1baa758db1a82ca2a3 2024-11-22T03:40:05,739 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/.tmp/A/d6f08aef05934343b1f738a5442203b6 as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/A/d6f08aef05934343b1f738a5442203b6 2024-11-22T03:40:05,740 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42545 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-22T03:40:05,740 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4. 2024-11-22T03:40:05,740 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4. as already flushing 2024-11-22T03:40:05,740 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4. 2024-11-22T03:40:05,741 ERROR [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => f970bd9f1e861e2b64a2fb87fb209ac4, NAME => 'TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:40:05,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => f970bd9f1e861e2b64a2fb87fb209ac4, NAME => 'TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:40:05,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42627 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f970bd9f1e861e2b64a2fb87fb209ac4, NAME => 'TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f970bd9f1e861e2b64a2fb87fb209ac4, NAME => 'TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:40:05,763 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/A/d6f08aef05934343b1f738a5442203b6, entries=200, sequenceid=333, filesize=14.4 K 2024-11-22T03:40:05,764 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/.tmp/B/549bca3e86484bba81f9a9d7402f3d35 as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/B/549bca3e86484bba81f9a9d7402f3d35 2024-11-22T03:40:05,786 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/B/549bca3e86484bba81f9a9d7402f3d35, entries=150, sequenceid=333, filesize=12.0 K 2024-11-22T03:40:05,788 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/.tmp/C/e15a26d1669c4c1baa758db1a82ca2a3 as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/C/e15a26d1669c4c1baa758db1a82ca2a3 2024-11-22T03:40:05,794 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/C/e15a26d1669c4c1baa758db1a82ca2a3, entries=150, sequenceid=333, filesize=12.0 K 2024-11-22T03:40:05,795 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=127.47 KB/130530 for f970bd9f1e861e2b64a2fb87fb209ac4 in 543ms, sequenceid=333, compaction requested=false 2024-11-22T03:40:05,795 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f970bd9f1e861e2b64a2fb87fb209ac4: 2024-11-22T03:40:05,896 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42545 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-22T03:40:05,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4. 2024-11-22T03:40:05,896 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2902): Flushing f970bd9f1e861e2b64a2fb87fb209ac4 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-22T03:40:05,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-22T03:40:05,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/.tmp/A/c0bd9fec57124d94b2204832a0dea86b is 50, key is test_row_0/A:col10/1732246805285/Put/seqid=0 2024-11-22T03:40:05,926 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4. as already flushing 2024-11-22T03:40:05,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(8855): Flush requested on f970bd9f1e861e2b64a2fb87fb209ac4 2024-11-22T03:40:05,948 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:05,949 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:05,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49612 deadline: 1732246865941, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:05,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49566 deadline: 1732246865941, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:05,949 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:05,949 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:05,950 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:05,949 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:05,949 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:05,950 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:05,950 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:40:05,950 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:40:05,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49606 deadline: 1732246865943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:05,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49582 deadline: 1732246865943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:05,950 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:05,950 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:05,950 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:40:05,950 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:05,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49590 deadline: 1732246865945, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:05,951 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:05,951 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:05,951 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:40:05,951 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:05,951 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:05,951 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:40:05,967 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073741909_1085 (size=12301) 2024-11-22T03:40:06,062 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:06,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49566 deadline: 1732246866061, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:06,062 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:06,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49606 deadline: 1732246866062, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:06,063 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:06,063 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:06,063 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:40:06,063 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:06,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49582 deadline: 1732246866062, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:06,064 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:06,064 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:06,064 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:40:06,064 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:06,064 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:06,064 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:06,064 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:40:06,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49612 deadline: 1732246866062, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:06,065 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:06,064 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:06,065 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:06,065 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:40:06,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49590 deadline: 1732246866062, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:06,065 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:06,065 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:06,065 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:40:06,271 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:06,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49582 deadline: 1732246866269, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:06,272 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:06,272 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:06,272 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:06,272 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:40:06,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49566 deadline: 1732246866269, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:06,272 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:06,272 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:06,272 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:06,272 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:40:06,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49606 deadline: 1732246866269, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:06,274 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:06,274 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:06,274 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:06,274 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:40:06,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49612 deadline: 1732246866270, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:06,274 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:06,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49590 deadline: 1732246866270, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:06,275 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:06,276 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:06,276 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:40:06,277 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:06,277 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:06,277 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:40:06,367 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=357 (bloomFilter=true), to=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/.tmp/A/c0bd9fec57124d94b2204832a0dea86b 2024-11-22T03:40:06,403 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/.tmp/B/e2f8b73af76f4802822b4a935de88c17 is 50, key is test_row_0/B:col10/1732246805285/Put/seqid=0 2024-11-22T03:40:06,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-22T03:40:06,474 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073741910_1086 (size=12301) 2024-11-22T03:40:06,475 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=357 (bloomFilter=true), to=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/.tmp/B/e2f8b73af76f4802822b4a935de88c17 2024-11-22T03:40:06,503 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/.tmp/C/b2dd03f386d44f348130c9816093e5b6 is 50, key is test_row_0/C:col10/1732246805285/Put/seqid=0 2024-11-22T03:40:06,548 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073741911_1087 (size=12301) 2024-11-22T03:40:06,584 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:06,584 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:06,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49566 deadline: 1732246866579, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:06,585 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:06,586 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:06,585 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:06,586 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:40:06,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49606 deadline: 1732246866579, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:06,586 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:06,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49590 deadline: 1732246866579, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:06,586 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:06,586 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:06,586 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:40:06,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49582 deadline: 1732246866579, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:06,586 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:06,586 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:06,587 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:40:06,587 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:06,587 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:06,587 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:40:06,592 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:06,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49612 deadline: 1732246866589, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:06,594 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:06,594 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:06,594 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:40:06,950 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=357 (bloomFilter=true), to=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/.tmp/C/b2dd03f386d44f348130c9816093e5b6 2024-11-22T03:40:06,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/.tmp/A/c0bd9fec57124d94b2204832a0dea86b as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/A/c0bd9fec57124d94b2204832a0dea86b 2024-11-22T03:40:06,969 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/A/c0bd9fec57124d94b2204832a0dea86b, entries=150, sequenceid=357, filesize=12.0 K 2024-11-22T03:40:06,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/.tmp/B/e2f8b73af76f4802822b4a935de88c17 as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/B/e2f8b73af76f4802822b4a935de88c17 2024-11-22T03:40:06,981 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/B/e2f8b73af76f4802822b4a935de88c17, entries=150, sequenceid=357, filesize=12.0 K 2024-11-22T03:40:06,984 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/.tmp/C/b2dd03f386d44f348130c9816093e5b6 as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/C/b2dd03f386d44f348130c9816093e5b6 2024-11-22T03:40:06,993 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/C/b2dd03f386d44f348130c9816093e5b6, entries=150, sequenceid=357, filesize=12.0 K 2024-11-22T03:40:06,994 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=73.80 KB/75570 for f970bd9f1e861e2b64a2fb87fb209ac4 in 1098ms, sequenceid=357, compaction requested=true 2024-11-22T03:40:06,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2603): Flush status journal for f970bd9f1e861e2b64a2fb87fb209ac4: 2024-11-22T03:40:06,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4. 2024-11-22T03:40:06,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=22 2024-11-22T03:40:06,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42627 {}] master.HMaster(4169): Remote procedure done, pid=22 2024-11-22T03:40:07,003 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=22, resume processing ppid=21 2024-11-22T03:40:07,003 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=22, ppid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7220 sec 2024-11-22T03:40:07,011 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees in 1.7350 sec 2024-11-22T03:40:07,094 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-22T03:40:07,094 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f970bd9f1e861e2b64a2fb87fb209ac4 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-22T03:40:07,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(8855): Flush requested on f970bd9f1e861e2b64a2fb87fb209ac4 2024-11-22T03:40:07,107 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/.tmp/A/5818145b02e14d9f9c0b2427280fb481 is 50, key is test_row_0/A:col10/1732246805941/Put/seqid=0 2024-11-22T03:40:07,142 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:07,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49590 deadline: 1732246867134, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:07,143 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:07,143 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:07,143 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:40:07,144 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:07,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49566 deadline: 1732246867137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:07,145 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:07,145 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:07,145 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:40:07,146 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073741912_1088 (size=14741) 2024-11-22T03:40:07,147 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:07,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49612 deadline: 1732246867139, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:07,149 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=372 (bloomFilter=true), to=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/.tmp/A/5818145b02e14d9f9c0b2427280fb481 2024-11-22T03:40:07,151 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:07,151 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:07,151 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:07,151 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:40:07,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49582 deadline: 1732246867142, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:07,151 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:07,152 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:07,152 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:07,152 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:40:07,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49606 deadline: 1732246867144, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:07,152 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:07,152 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:07,152 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:40:07,171 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/.tmp/B/72bf86328be448638eaa8717eaa54a26 is 50, key is test_row_0/B:col10/1732246805941/Put/seqid=0 2024-11-22T03:40:07,202 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073741913_1089 (size=12301) 2024-11-22T03:40:07,203 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=372 (bloomFilter=true), to=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/.tmp/B/72bf86328be448638eaa8717eaa54a26 2024-11-22T03:40:07,223 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/.tmp/C/3f329e65bc024cb0916a0b3b969f4fb2 is 50, key is test_row_0/C:col10/1732246805941/Put/seqid=0 2024-11-22T03:40:07,252 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:07,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49590 deadline: 1732246867251, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:07,252 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:07,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49566 deadline: 1732246867251, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:07,253 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:07,253 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:07,253 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:40:07,253 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:07,253 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:07,253 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:40:07,270 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073741914_1090 (size=12301) 2024-11-22T03:40:07,271 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:07,271 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:07,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49582 deadline: 1732246867259, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:07,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49612 deadline: 1732246867259, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:07,272 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:07,272 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=372 (bloomFilter=true), to=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/.tmp/C/3f329e65bc024cb0916a0b3b969f4fb2 2024-11-22T03:40:07,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49606 deadline: 1732246867259, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:07,272 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:07,272 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:07,272 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:40:07,273 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:07,273 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:07,273 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:40:07,280 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/.tmp/A/5818145b02e14d9f9c0b2427280fb481 as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/A/5818145b02e14d9f9c0b2427280fb481 2024-11-22T03:40:07,287 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:07,288 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:07,288 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:40:07,291 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/A/5818145b02e14d9f9c0b2427280fb481, entries=200, sequenceid=372, filesize=14.4 K 2024-11-22T03:40:07,292 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/.tmp/B/72bf86328be448638eaa8717eaa54a26 as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/B/72bf86328be448638eaa8717eaa54a26 2024-11-22T03:40:07,300 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/B/72bf86328be448638eaa8717eaa54a26, entries=150, sequenceid=372, filesize=12.0 K 2024-11-22T03:40:07,302 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/.tmp/C/3f329e65bc024cb0916a0b3b969f4fb2 as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/C/3f329e65bc024cb0916a0b3b969f4fb2 2024-11-22T03:40:07,312 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/C/3f329e65bc024cb0916a0b3b969f4fb2, entries=150, sequenceid=372, filesize=12.0 K 2024-11-22T03:40:07,314 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=127.47 KB/130530 for f970bd9f1e861e2b64a2fb87fb209ac4 in 220ms, sequenceid=372, compaction requested=true 2024-11-22T03:40:07,314 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f970bd9f1e861e2b64a2fb87fb209ac4: 2024-11-22T03:40:07,314 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-22T03:40:07,315 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f970bd9f1e861e2b64a2fb87fb209ac4:A, priority=-2147483648, current under compaction store size is 1 2024-11-22T03:40:07,315 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T03:40:07,316 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 54834 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-22T03:40:07,316 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.HStore(1541): f970bd9f1e861e2b64a2fb87fb209ac4/A is initiating minor compaction (all files) 2024-11-22T03:40:07,316 INFO [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f970bd9f1e861e2b64a2fb87fb209ac4/A in TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4. 2024-11-22T03:40:07,316 INFO [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/A/d405092cdcbd408caf5f8951e21805c3, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/A/d6f08aef05934343b1f738a5442203b6, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/A/c0bd9fec57124d94b2204832a0dea86b, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/A/5818145b02e14d9f9c0b2427280fb481] into tmpdir=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/.tmp, totalSize=53.5 K 2024-11-22T03:40:07,318 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-22T03:40:07,318 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] compactions.Compactor(225): Compacting d405092cdcbd408caf5f8951e21805c3, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=317, earliestPutTs=1732246804785 2024-11-22T03:40:07,319 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] compactions.Compactor(225): Compacting d6f08aef05934343b1f738a5442203b6, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=333, earliestPutTs=1732246804909 2024-11-22T03:40:07,320 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] compactions.Compactor(225): Compacting c0bd9fec57124d94b2204832a0dea86b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=357, earliestPutTs=1732246805265 2024-11-22T03:40:07,320 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49954 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-22T03:40:07,320 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HStore(1541): f970bd9f1e861e2b64a2fb87fb209ac4/B is initiating minor compaction (all files) 2024-11-22T03:40:07,320 INFO [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f970bd9f1e861e2b64a2fb87fb209ac4/B in TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4. 2024-11-22T03:40:07,321 INFO [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/B/7e919651fda74c34b20c4a3f74ef0026, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/B/549bca3e86484bba81f9a9d7402f3d35, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/B/e2f8b73af76f4802822b4a935de88c17, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/B/72bf86328be448638eaa8717eaa54a26] into tmpdir=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/.tmp, totalSize=48.8 K 2024-11-22T03:40:07,321 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5818145b02e14d9f9c0b2427280fb481, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=372, earliestPutTs=1732246805939 2024-11-22T03:40:07,321 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] compactions.Compactor(225): Compacting 7e919651fda74c34b20c4a3f74ef0026, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=317, earliestPutTs=1732246804785 2024-11-22T03:40:07,322 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] compactions.Compactor(225): Compacting 549bca3e86484bba81f9a9d7402f3d35, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=333, earliestPutTs=1732246804909 2024-11-22T03:40:07,321 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f970bd9f1e861e2b64a2fb87fb209ac4:B, priority=-2147483648, current under compaction store size is 2 2024-11-22T03:40:07,322 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T03:40:07,322 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] compactions.Compactor(225): Compacting e2f8b73af76f4802822b4a935de88c17, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=357, earliestPutTs=1732246805265 2024-11-22T03:40:07,323 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] compactions.Compactor(225): Compacting 72bf86328be448638eaa8717eaa54a26, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=372, earliestPutTs=1732246805941 2024-11-22T03:40:07,325 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f970bd9f1e861e2b64a2fb87fb209ac4:C, priority=-2147483648, current under compaction store size is 3 2024-11-22T03:40:07,328 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T03:40:07,347 INFO [RS:0;c85114ed5096:42545-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f970bd9f1e861e2b64a2fb87fb209ac4#A#compaction#78 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T03:40:07,348 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/.tmp/A/8102c612cad44458827ed502b2e65903 is 50, key is test_row_0/A:col10/1732246805941/Put/seqid=0 2024-11-22T03:40:07,369 INFO [RS:0;c85114ed5096:42545-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f970bd9f1e861e2b64a2fb87fb209ac4#B#compaction#79 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T03:40:07,370 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/.tmp/B/0950866217f9482c88e3e608290ecd66 is 50, key is test_row_0/B:col10/1732246805941/Put/seqid=0 2024-11-22T03:40:07,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-22T03:40:07,419 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073741915_1091 (size=13187) 2024-11-22T03:40:07,419 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-22T03:40:07,420 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42627 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-22T03:40:07,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42627 {}] procedure2.ProcedureExecutor(1139): Stored pid=23, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees 2024-11-22T03:40:07,423 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=23, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-22T03:40:07,425 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=23, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-22T03:40:07,425 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=24, ppid=23, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-22T03:40:07,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-22T03:40:07,438 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073741916_1092 (size=13187) 2024-11-22T03:40:07,440 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/.tmp/A/8102c612cad44458827ed502b2e65903 as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/A/8102c612cad44458827ed502b2e65903 2024-11-22T03:40:07,454 INFO [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in f970bd9f1e861e2b64a2fb87fb209ac4/A of f970bd9f1e861e2b64a2fb87fb209ac4 into 8102c612cad44458827ed502b2e65903(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T03:40:07,455 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f970bd9f1e861e2b64a2fb87fb209ac4: 2024-11-22T03:40:07,455 INFO [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., storeName=f970bd9f1e861e2b64a2fb87fb209ac4/A, priority=12, startTime=1732246807314; duration=0sec 2024-11-22T03:40:07,455 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T03:40:07,455 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f970bd9f1e861e2b64a2fb87fb209ac4:A 2024-11-22T03:40:07,455 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-22T03:40:07,457 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49954 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-22T03:40:07,457 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.HStore(1541): f970bd9f1e861e2b64a2fb87fb209ac4/C is initiating minor compaction (all files) 2024-11-22T03:40:07,457 INFO [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f970bd9f1e861e2b64a2fb87fb209ac4/C in TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4. 2024-11-22T03:40:07,458 INFO [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/C/3b798870b3c44845916d55d82f80abb4, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/C/e15a26d1669c4c1baa758db1a82ca2a3, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/C/b2dd03f386d44f348130c9816093e5b6, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/C/3f329e65bc024cb0916a0b3b969f4fb2] into tmpdir=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/.tmp, totalSize=48.8 K 2024-11-22T03:40:07,458 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3b798870b3c44845916d55d82f80abb4, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=317, earliestPutTs=1732246804785 2024-11-22T03:40:07,460 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] compactions.Compactor(225): Compacting e15a26d1669c4c1baa758db1a82ca2a3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=333, earliestPutTs=1732246804909 2024-11-22T03:40:07,460 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] compactions.Compactor(225): Compacting b2dd03f386d44f348130c9816093e5b6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=357, earliestPutTs=1732246805265 2024-11-22T03:40:07,461 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3f329e65bc024cb0916a0b3b969f4fb2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=372, earliestPutTs=1732246805941 2024-11-22T03:40:07,464 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/.tmp/B/0950866217f9482c88e3e608290ecd66 as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/B/0950866217f9482c88e3e608290ecd66 2024-11-22T03:40:07,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(8855): Flush requested on f970bd9f1e861e2b64a2fb87fb209ac4 2024-11-22T03:40:07,468 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-22T03:40:07,468 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f970bd9f1e861e2b64a2fb87fb209ac4 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-22T03:40:07,473 INFO [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in f970bd9f1e861e2b64a2fb87fb209ac4/B of f970bd9f1e861e2b64a2fb87fb209ac4 into 0950866217f9482c88e3e608290ecd66(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T03:40:07,473 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f970bd9f1e861e2b64a2fb87fb209ac4: 2024-11-22T03:40:07,473 INFO [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., storeName=f970bd9f1e861e2b64a2fb87fb209ac4/B, priority=12, startTime=1732246807315; duration=0sec 2024-11-22T03:40:07,474 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T03:40:07,474 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f970bd9f1e861e2b64a2fb87fb209ac4:B 2024-11-22T03:40:07,497 INFO [RS:0;c85114ed5096:42545-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f970bd9f1e861e2b64a2fb87fb209ac4#C#compaction#80 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T03:40:07,498 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/.tmp/C/60fdfd0728e94e19b2cbd98d852f28b8 is 50, key is test_row_0/C:col10/1732246805941/Put/seqid=0 2024-11-22T03:40:07,502 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:07,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49590 deadline: 1732246867493, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:07,503 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:07,503 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:07,503 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:40:07,507 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/.tmp/A/19d5c76c52cb46789f6bf5885b7650e4 is 50, key is test_row_0/A:col10/1732246807466/Put/seqid=0 2024-11-22T03:40:07,511 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:07,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49612 deadline: 1732246867500, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:07,512 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:07,512 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:07,512 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:07,512 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:40:07,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49566 deadline: 1732246867501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:07,512 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:07,512 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:07,512 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:07,512 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:40:07,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49582 deadline: 1732246867502, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:07,513 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:07,513 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:07,513 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:40:07,513 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:07,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49606 deadline: 1732246867502, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:07,515 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:07,515 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:07,515 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:40:07,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-22T03:40:07,551 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073741917_1093 (size=13187) 2024-11-22T03:40:07,563 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073741918_1094 (size=12301) 2024-11-22T03:40:07,564 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=397 (bloomFilter=true), to=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/.tmp/A/19d5c76c52cb46789f6bf5885b7650e4 2024-11-22T03:40:07,580 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42545 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-22T03:40:07,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4. 2024-11-22T03:40:07,581 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4. as already flushing 2024-11-22T03:40:07,581 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4. 2024-11-22T03:40:07,581 ERROR [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => f970bd9f1e861e2b64a2fb87fb209ac4, NAME => 'TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:40:07,581 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => f970bd9f1e861e2b64a2fb87fb209ac4, NAME => 'TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:40:07,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42627 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f970bd9f1e861e2b64a2fb87fb209ac4, NAME => 'TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f970bd9f1e861e2b64a2fb87fb209ac4, NAME => 'TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:40:07,594 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/.tmp/B/eb8039590ba24f66984c65a34d966d6d is 50, key is test_row_0/B:col10/1732246807466/Put/seqid=0 2024-11-22T03:40:07,614 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:07,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49590 deadline: 1732246867609, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:07,615 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:07,615 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:07,615 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:40:07,621 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:07,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49566 deadline: 1732246867619, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:07,622 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:07,622 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:07,622 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:07,622 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:40:07,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49606 deadline: 1732246867620, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:07,623 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:07,623 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:07,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49582 deadline: 1732246867619, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:07,623 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:07,623 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:40:07,623 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:07,623 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:07,623 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:40:07,644 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073741919_1095 (size=12301) 2024-11-22T03:40:07,645 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=397 (bloomFilter=true), to=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/.tmp/B/eb8039590ba24f66984c65a34d966d6d 2024-11-22T03:40:07,667 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/.tmp/C/dd8cd07fc06040c98cc9a47155304fa0 is 50, key is test_row_0/C:col10/1732246807466/Put/seqid=0 2024-11-22T03:40:07,706 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073741920_1096 (size=12301) 2024-11-22T03:40:07,714 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=397 (bloomFilter=true), to=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/.tmp/C/dd8cd07fc06040c98cc9a47155304fa0 2024-11-22T03:40:07,726 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/.tmp/A/19d5c76c52cb46789f6bf5885b7650e4 as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/A/19d5c76c52cb46789f6bf5885b7650e4 2024-11-22T03:40:07,734 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42545 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-22T03:40:07,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4. 2024-11-22T03:40:07,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4. as already flushing 2024-11-22T03:40:07,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4. 2024-11-22T03:40:07,735 ERROR [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => f970bd9f1e861e2b64a2fb87fb209ac4, NAME => 'TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:40:07,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => f970bd9f1e861e2b64a2fb87fb209ac4, NAME => 'TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:40:07,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42627 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f970bd9f1e861e2b64a2fb87fb209ac4, NAME => 'TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f970bd9f1e861e2b64a2fb87fb209ac4, NAME => 'TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:40:07,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-22T03:40:07,740 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/A/19d5c76c52cb46789f6bf5885b7650e4, entries=150, sequenceid=397, filesize=12.0 K 2024-11-22T03:40:07,742 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/.tmp/B/eb8039590ba24f66984c65a34d966d6d as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/B/eb8039590ba24f66984c65a34d966d6d 2024-11-22T03:40:07,749 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/B/eb8039590ba24f66984c65a34d966d6d, entries=150, sequenceid=397, filesize=12.0 K 2024-11-22T03:40:07,762 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/.tmp/C/dd8cd07fc06040c98cc9a47155304fa0 as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/C/dd8cd07fc06040c98cc9a47155304fa0 2024-11-22T03:40:07,771 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/C/dd8cd07fc06040c98cc9a47155304fa0, entries=150, sequenceid=397, filesize=12.0 K 2024-11-22T03:40:07,773 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for f970bd9f1e861e2b64a2fb87fb209ac4 in 305ms, sequenceid=397, compaction requested=false 2024-11-22T03:40:07,773 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f970bd9f1e861e2b64a2fb87fb209ac4: 2024-11-22T03:40:07,825 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-22T03:40:07,826 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f970bd9f1e861e2b64a2fb87fb209ac4 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-22T03:40:07,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(8855): Flush requested on f970bd9f1e861e2b64a2fb87fb209ac4 2024-11-22T03:40:07,850 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/.tmp/A/346aa3a7f4c340d2bd5f2f50473fec1d is 50, key is test_row_0/A:col10/1732246807500/Put/seqid=0 2024-11-22T03:40:07,886 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073741921_1097 (size=14741) 2024-11-22T03:40:07,889 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42545 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-22T03:40:07,890 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=412 (bloomFilter=true), to=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/.tmp/A/346aa3a7f4c340d2bd5f2f50473fec1d 2024-11-22T03:40:07,890 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4. 2024-11-22T03:40:07,890 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4. as already flushing 2024-11-22T03:40:07,890 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4. 2024-11-22T03:40:07,890 ERROR [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => f970bd9f1e861e2b64a2fb87fb209ac4, NAME => 'TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:40:07,890 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => f970bd9f1e861e2b64a2fb87fb209ac4, NAME => 'TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:40:07,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42627 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f970bd9f1e861e2b64a2fb87fb209ac4, NAME => 'TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f970bd9f1e861e2b64a2fb87fb209ac4, NAME => 'TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:40:07,898 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:07,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49590 deadline: 1732246867886, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:07,898 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:07,898 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:07,899 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:40:07,899 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:07,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49612 deadline: 1732246867888, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:07,900 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:07,900 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:07,900 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:40:07,900 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:07,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49606 deadline: 1732246867889, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:07,901 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:07,901 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:07,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49566 deadline: 1732246867894, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:07,901 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:07,901 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:40:07,901 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:07,901 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:07,901 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:40:07,906 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:07,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49582 deadline: 1732246867898, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:07,906 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:07,906 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:07,906 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:40:07,919 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/.tmp/B/d4890a262733475693d2f4112a05d516 is 50, key is test_row_0/B:col10/1732246807500/Put/seqid=0 2024-11-22T03:40:07,961 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073741922_1098 (size=12301) 2024-11-22T03:40:07,963 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/.tmp/C/60fdfd0728e94e19b2cbd98d852f28b8 as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/C/60fdfd0728e94e19b2cbd98d852f28b8 2024-11-22T03:40:07,966 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=412 (bloomFilter=true), to=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/.tmp/B/d4890a262733475693d2f4112a05d516 2024-11-22T03:40:07,983 INFO [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in f970bd9f1e861e2b64a2fb87fb209ac4/C of f970bd9f1e861e2b64a2fb87fb209ac4 into 60fdfd0728e94e19b2cbd98d852f28b8(size=12.9 K), total size for store is 24.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T03:40:07,983 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f970bd9f1e861e2b64a2fb87fb209ac4: 2024-11-22T03:40:07,983 INFO [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., storeName=f970bd9f1e861e2b64a2fb87fb209ac4/C, priority=12, startTime=1732246807325; duration=0sec 2024-11-22T03:40:07,983 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T03:40:07,983 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f970bd9f1e861e2b64a2fb87fb209ac4:C 2024-11-22T03:40:07,995 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/.tmp/C/c6e1e26de0964d538ca712887fcc5a2f is 50, key is test_row_0/C:col10/1732246807500/Put/seqid=0 2024-11-22T03:40:08,018 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:08,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49566 deadline: 1732246868009, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:08,018 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:08,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49590 deadline: 1732246868010, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:08,019 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:08,019 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:08,019 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:40:08,020 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:08,020 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:08,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49612 deadline: 1732246868010, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:08,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49582 deadline: 1732246868010, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:08,021 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:08,021 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:08,021 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:40:08,014 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:08,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49606 deadline: 1732246868009, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:08,029 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:08,029 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:08,029 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:08,029 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:08,029 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:40:08,029 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:40:08,029 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:08,029 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:08,029 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:40:08,043 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42545 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-22T03:40:08,044 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4. 2024-11-22T03:40:08,044 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4. as already flushing 2024-11-22T03:40:08,044 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4. 2024-11-22T03:40:08,044 ERROR [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => f970bd9f1e861e2b64a2fb87fb209ac4, NAME => 'TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:40:08,044 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => f970bd9f1e861e2b64a2fb87fb209ac4, NAME => 'TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:40:08,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42627 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f970bd9f1e861e2b64a2fb87fb209ac4, NAME => 'TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f970bd9f1e861e2b64a2fb87fb209ac4, NAME => 'TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:40:08,046 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073741923_1099 (size=12301) 2024-11-22T03:40:08,049 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=412 (bloomFilter=true), to=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/.tmp/C/c6e1e26de0964d538ca712887fcc5a2f 2024-11-22T03:40:08,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-22T03:40:08,064 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/.tmp/A/346aa3a7f4c340d2bd5f2f50473fec1d as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/A/346aa3a7f4c340d2bd5f2f50473fec1d 2024-11-22T03:40:08,077 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/A/346aa3a7f4c340d2bd5f2f50473fec1d, entries=200, sequenceid=412, filesize=14.4 K 2024-11-22T03:40:08,078 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/.tmp/B/d4890a262733475693d2f4112a05d516 as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/B/d4890a262733475693d2f4112a05d516 2024-11-22T03:40:08,088 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/B/d4890a262733475693d2f4112a05d516, entries=150, sequenceid=412, filesize=12.0 K 2024-11-22T03:40:08,089 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/.tmp/C/c6e1e26de0964d538ca712887fcc5a2f as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/C/c6e1e26de0964d538ca712887fcc5a2f 2024-11-22T03:40:08,105 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/C/c6e1e26de0964d538ca712887fcc5a2f, entries=150, sequenceid=412, filesize=12.0 K 2024-11-22T03:40:08,107 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for f970bd9f1e861e2b64a2fb87fb209ac4 in 280ms, sequenceid=412, compaction requested=true 2024-11-22T03:40:08,107 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f970bd9f1e861e2b64a2fb87fb209ac4: 2024-11-22T03:40:08,107 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f970bd9f1e861e2b64a2fb87fb209ac4:A, priority=-2147483648, current under compaction store size is 1 2024-11-22T03:40:08,107 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T03:40:08,107 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T03:40:08,107 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T03:40:08,107 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f970bd9f1e861e2b64a2fb87fb209ac4:B, priority=-2147483648, current under compaction store size is 2 2024-11-22T03:40:08,108 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T03:40:08,108 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f970bd9f1e861e2b64a2fb87fb209ac4:C, priority=-2147483648, current under compaction store size is 3 2024-11-22T03:40:08,108 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T03:40:08,108 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37789 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T03:40:08,108 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40229 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T03:40:08,108 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.HStore(1541): f970bd9f1e861e2b64a2fb87fb209ac4/A is initiating minor compaction (all files) 2024-11-22T03:40:08,108 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HStore(1541): f970bd9f1e861e2b64a2fb87fb209ac4/B is initiating minor compaction (all files) 2024-11-22T03:40:08,108 INFO [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f970bd9f1e861e2b64a2fb87fb209ac4/B in TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4. 2024-11-22T03:40:08,109 INFO [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f970bd9f1e861e2b64a2fb87fb209ac4/A in TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4. 2024-11-22T03:40:08,109 INFO [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/A/8102c612cad44458827ed502b2e65903, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/A/19d5c76c52cb46789f6bf5885b7650e4, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/A/346aa3a7f4c340d2bd5f2f50473fec1d] into tmpdir=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/.tmp, totalSize=39.3 K 2024-11-22T03:40:08,109 INFO [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/B/0950866217f9482c88e3e608290ecd66, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/B/eb8039590ba24f66984c65a34d966d6d, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/B/d4890a262733475693d2f4112a05d516] into tmpdir=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/.tmp, totalSize=36.9 K 2024-11-22T03:40:08,109 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8102c612cad44458827ed502b2e65903, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=372, earliestPutTs=1732246805941 2024-11-22T03:40:08,109 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] compactions.Compactor(225): Compacting 0950866217f9482c88e3e608290ecd66, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=372, earliestPutTs=1732246805941 2024-11-22T03:40:08,110 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] compactions.Compactor(225): Compacting 19d5c76c52cb46789f6bf5885b7650e4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=397, earliestPutTs=1732246807115 2024-11-22T03:40:08,117 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] compactions.Compactor(225): Compacting eb8039590ba24f66984c65a34d966d6d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=397, earliestPutTs=1732246807115 2024-11-22T03:40:08,117 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] compactions.Compactor(225): Compacting 346aa3a7f4c340d2bd5f2f50473fec1d, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=412, earliestPutTs=1732246807497 2024-11-22T03:40:08,119 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] compactions.Compactor(225): Compacting d4890a262733475693d2f4112a05d516, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=412, earliestPutTs=1732246807500 2024-11-22T03:40:08,149 INFO [RS:0;c85114ed5096:42545-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f970bd9f1e861e2b64a2fb87fb209ac4#B#compaction#88 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-22T03:40:08,150 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/.tmp/B/3a5fb2435eae4c51ba4d913077c59c0e is 50, key is test_row_0/B:col10/1732246807500/Put/seqid=0 2024-11-22T03:40:08,151 INFO [RS:0;c85114ed5096:42545-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f970bd9f1e861e2b64a2fb87fb209ac4#A#compaction#87 average throughput is 1.31 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T03:40:08,151 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/.tmp/A/7409d20987cb4ff8b7bf232924241410 is 50, key is test_row_0/A:col10/1732246807500/Put/seqid=0 2024-11-22T03:40:08,167 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073741924_1100 (size=13289) 2024-11-22T03:40:08,181 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073741925_1101 (size=13289) 2024-11-22T03:40:08,197 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42545 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-22T03:40:08,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4. 2024-11-22T03:40:08,198 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2902): Flushing f970bd9f1e861e2b64a2fb87fb209ac4 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-22T03:40:08,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/.tmp/A/96968a6efacc4168b786ad9f0419e2d2 is 50, key is test_row_0/A:col10/1732246807886/Put/seqid=0 2024-11-22T03:40:08,224 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073741926_1102 (size=12301) 2024-11-22T03:40:08,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(8855): Flush requested on f970bd9f1e861e2b64a2fb87fb209ac4 2024-11-22T03:40:08,233 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4. as already flushing 2024-11-22T03:40:08,254 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:08,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49582 deadline: 1732246868243, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:08,255 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:08,255 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:08,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49612 deadline: 1732246868251, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:08,255 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:08,255 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:40:08,256 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:08,256 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:08,256 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:08,256 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:40:08,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49590 deadline: 1732246868252, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:08,257 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:08,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49566 deadline: 1732246868254, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:08,257 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:08,257 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:08,258 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:40:08,257 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:08,258 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:08,258 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:40:08,259 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:08,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49606 deadline: 1732246868255, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:08,260 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:08,260 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:08,260 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:40:08,362 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:08,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49612 deadline: 1732246868358, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:08,362 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:08,362 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:08,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49582 deadline: 1732246868358, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:08,363 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:08,363 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:40:08,363 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:08,363 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:08,363 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:40:08,372 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:08,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49590 deadline: 1732246868369, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:08,373 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:08,374 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:08,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49566 deadline: 1732246868369, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:08,374 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:08,374 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:40:08,374 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:08,374 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:08,374 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:08,374 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:40:08,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49606 deadline: 1732246868369, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:08,374 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:08,374 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:08,374 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:40:08,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-22T03:40:08,569 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:08,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49582 deadline: 1732246868569, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:08,572 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:08,571 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:08,572 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:08,572 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:40:08,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49612 deadline: 1732246868569, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:08,573 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:08,573 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:08,573 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:40:08,581 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:08,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49590 deadline: 1732246868579, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:08,582 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:08,582 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:08,582 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:08,582 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:40:08,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49566 deadline: 1732246868580, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:08,583 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:08,583 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:08,583 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:40:08,583 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:08,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49606 deadline: 1732246868580, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:08,584 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:08,584 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:08,584 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:40:08,592 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/.tmp/B/3a5fb2435eae4c51ba4d913077c59c0e as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/B/3a5fb2435eae4c51ba4d913077c59c0e 2024-11-22T03:40:08,592 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/.tmp/A/7409d20987cb4ff8b7bf232924241410 as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/A/7409d20987cb4ff8b7bf232924241410 2024-11-22T03:40:08,602 INFO [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f970bd9f1e861e2b64a2fb87fb209ac4/A of f970bd9f1e861e2b64a2fb87fb209ac4 into 7409d20987cb4ff8b7bf232924241410(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T03:40:08,602 INFO [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f970bd9f1e861e2b64a2fb87fb209ac4/B of f970bd9f1e861e2b64a2fb87fb209ac4 into 3a5fb2435eae4c51ba4d913077c59c0e(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T03:40:08,602 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f970bd9f1e861e2b64a2fb87fb209ac4: 2024-11-22T03:40:08,602 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f970bd9f1e861e2b64a2fb87fb209ac4: 2024-11-22T03:40:08,602 INFO [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., storeName=f970bd9f1e861e2b64a2fb87fb209ac4/A, priority=13, startTime=1732246808107; duration=0sec 2024-11-22T03:40:08,602 INFO [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., storeName=f970bd9f1e861e2b64a2fb87fb209ac4/B, priority=13, startTime=1732246808107; duration=0sec 2024-11-22T03:40:08,602 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T03:40:08,602 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f970bd9f1e861e2b64a2fb87fb209ac4:B 2024-11-22T03:40:08,602 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T03:40:08,602 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f970bd9f1e861e2b64a2fb87fb209ac4:A 2024-11-22T03:40:08,602 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T03:40:08,604 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37789 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T03:40:08,604 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HStore(1541): f970bd9f1e861e2b64a2fb87fb209ac4/C is initiating minor compaction (all files) 2024-11-22T03:40:08,604 INFO [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f970bd9f1e861e2b64a2fb87fb209ac4/C in TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4. 2024-11-22T03:40:08,604 INFO [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/C/60fdfd0728e94e19b2cbd98d852f28b8, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/C/dd8cd07fc06040c98cc9a47155304fa0, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/C/c6e1e26de0964d538ca712887fcc5a2f] into tmpdir=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/.tmp, totalSize=36.9 K 2024-11-22T03:40:08,605 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] compactions.Compactor(225): Compacting 60fdfd0728e94e19b2cbd98d852f28b8, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=372, earliestPutTs=1732246805941 2024-11-22T03:40:08,606 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] compactions.Compactor(225): Compacting dd8cd07fc06040c98cc9a47155304fa0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=397, earliestPutTs=1732246807115 2024-11-22T03:40:08,606 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] compactions.Compactor(225): Compacting c6e1e26de0964d538ca712887fcc5a2f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=412, earliestPutTs=1732246807500 2024-11-22T03:40:08,625 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=435 (bloomFilter=true), to=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/.tmp/A/96968a6efacc4168b786ad9f0419e2d2 2024-11-22T03:40:08,627 INFO [RS:0;c85114ed5096:42545-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f970bd9f1e861e2b64a2fb87fb209ac4#C#compaction#90 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T03:40:08,629 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/.tmp/C/7f2dbb2fab82448dabdfdcad1b3d7749 is 50, key is test_row_0/C:col10/1732246807500/Put/seqid=0 2024-11-22T03:40:08,641 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/.tmp/B/85d6e91de066416c8b4b382e77605b84 is 50, key is test_row_0/B:col10/1732246807886/Put/seqid=0 2024-11-22T03:40:08,641 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073741927_1103 (size=13289) 2024-11-22T03:40:08,654 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/.tmp/C/7f2dbb2fab82448dabdfdcad1b3d7749 as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/C/7f2dbb2fab82448dabdfdcad1b3d7749 2024-11-22T03:40:08,667 INFO [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f970bd9f1e861e2b64a2fb87fb209ac4/C of f970bd9f1e861e2b64a2fb87fb209ac4 into 7f2dbb2fab82448dabdfdcad1b3d7749(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T03:40:08,667 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f970bd9f1e861e2b64a2fb87fb209ac4: 2024-11-22T03:40:08,667 INFO [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., storeName=f970bd9f1e861e2b64a2fb87fb209ac4/C, priority=13, startTime=1732246808108; duration=0sec 2024-11-22T03:40:08,667 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T03:40:08,667 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f970bd9f1e861e2b64a2fb87fb209ac4:C 2024-11-22T03:40:08,688 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073741928_1104 (size=12301) 2024-11-22T03:40:08,689 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=435 (bloomFilter=true), to=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/.tmp/B/85d6e91de066416c8b4b382e77605b84 2024-11-22T03:40:08,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/.tmp/C/fe573883a1474be38ffd221a6ef3d57d is 50, key is test_row_0/C:col10/1732246807886/Put/seqid=0 2024-11-22T03:40:08,709 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073741929_1105 (size=12301) 2024-11-22T03:40:08,879 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:08,879 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:08,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49612 deadline: 1732246868879, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:08,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49582 deadline: 1732246868879, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:08,880 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:08,880 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:08,880 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:40:08,880 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:08,880 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:08,880 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:40:08,890 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:08,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49590 deadline: 1732246868889, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:08,890 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:08,890 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:08,890 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:08,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49566 deadline: 1732246868889, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:08,890 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:40:08,890 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:08,891 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:08,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49606 deadline: 1732246868889, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:08,891 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:08,891 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:40:08,891 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:08,891 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:08,891 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:40:09,110 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=435 (bloomFilter=true), to=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/.tmp/C/fe573883a1474be38ffd221a6ef3d57d 2024-11-22T03:40:09,118 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/.tmp/A/96968a6efacc4168b786ad9f0419e2d2 as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/A/96968a6efacc4168b786ad9f0419e2d2 2024-11-22T03:40:09,127 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/A/96968a6efacc4168b786ad9f0419e2d2, entries=150, sequenceid=435, filesize=12.0 K 2024-11-22T03:40:09,129 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/.tmp/B/85d6e91de066416c8b4b382e77605b84 as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/B/85d6e91de066416c8b4b382e77605b84 2024-11-22T03:40:09,136 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/B/85d6e91de066416c8b4b382e77605b84, entries=150, sequenceid=435, filesize=12.0 K 2024-11-22T03:40:09,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/.tmp/C/fe573883a1474be38ffd221a6ef3d57d as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/C/fe573883a1474be38ffd221a6ef3d57d 2024-11-22T03:40:09,151 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/C/fe573883a1474be38ffd221a6ef3d57d, entries=150, sequenceid=435, filesize=12.0 K 2024-11-22T03:40:09,152 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=73.80 KB/75570 for f970bd9f1e861e2b64a2fb87fb209ac4 in 954ms, sequenceid=435, compaction requested=false 2024-11-22T03:40:09,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2603): Flush status journal for f970bd9f1e861e2b64a2fb87fb209ac4: 2024-11-22T03:40:09,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4. 2024-11-22T03:40:09,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=24 2024-11-22T03:40:09,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42627 {}] master.HMaster(4169): Remote procedure done, pid=24 2024-11-22T03:40:09,159 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=24, resume processing ppid=23 2024-11-22T03:40:09,159 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=24, ppid=23, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7300 sec 2024-11-22T03:40:09,162 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=23, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees in 1.7400 sec 2024-11-22T03:40:09,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(8855): Flush requested on f970bd9f1e861e2b64a2fb87fb209ac4 2024-11-22T03:40:09,390 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-22T03:40:09,391 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f970bd9f1e861e2b64a2fb87fb209ac4 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2024-11-22T03:40:09,397 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/.tmp/A/1012df2ff75844c08fdacb8cdef652f7 is 50, key is test_row_0/A:col10/1732246808240/Put/seqid=0 2024-11-22T03:40:09,418 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:09,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49582 deadline: 1732246869414, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:09,418 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:09,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49606 deadline: 1732246869414, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:09,419 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:09,419 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:09,419 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:09,419 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:40:09,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49590 deadline: 1732246869416, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:09,419 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:09,419 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:09,419 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:40:09,419 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:09,419 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:09,419 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:40:09,423 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:09,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49612 deadline: 1732246869418, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:09,423 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:09,424 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:09,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49566 deadline: 1732246869418, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:09,424 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:09,424 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:40:09,424 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:09,424 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:09,424 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:40:09,425 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073741930_1106 (size=17181) 2024-11-22T03:40:09,531 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:09,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49590 deadline: 1732246869529, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:09,532 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:09,532 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:09,532 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:09,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49582 deadline: 1732246869529, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:09,532 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:40:09,532 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:09,533 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:09,532 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:09,533 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:40:09,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49612 deadline: 1732246869529, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:09,533 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:09,533 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:09,533 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:09,533 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:40:09,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49606 deadline: 1732246869529, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:09,533 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:09,533 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:09,533 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:09,533 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:40:09,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49566 deadline: 1732246869529, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:09,534 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:09,534 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:09,534 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:40:09,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-22T03:40:09,579 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-22T03:40:09,579 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42627 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-22T03:40:09,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42627 {}] procedure2.ProcedureExecutor(1139): Stored pid=25, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees 2024-11-22T03:40:09,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-22T03:40:09,582 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=25, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-22T03:40:09,583 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=25, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-22T03:40:09,584 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=26, ppid=25, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-22T03:40:09,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-22T03:40:09,737 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42545 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-22T03:40:09,737 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4. 2024-11-22T03:40:09,738 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4. as already flushing 2024-11-22T03:40:09,738 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4. 2024-11-22T03:40:09,738 ERROR [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => f970bd9f1e861e2b64a2fb87fb209ac4, NAME => 'TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:40:09,738 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => f970bd9f1e861e2b64a2fb87fb209ac4, NAME => 'TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:40:09,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42627 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f970bd9f1e861e2b64a2fb87fb209ac4, NAME => 'TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f970bd9f1e861e2b64a2fb87fb209ac4, NAME => 'TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:40:09,742 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:09,743 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:09,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49566 deadline: 1732246869739, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:09,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49612 deadline: 1732246869739, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:09,743 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:09,743 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:09,743 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:09,743 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:40:09,743 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:09,743 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:40:09,744 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:09,744 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:09,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49590 deadline: 1732246869739, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:09,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49582 deadline: 1732246869739, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:09,744 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:09,744 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:09,744 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:09,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49606 deadline: 1732246869739, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:09,744 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:40:09,744 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:09,744 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:09,745 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:40:09,745 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:09,745 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:09,745 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:40:09,826 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=454 (bloomFilter=true), to=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/.tmp/A/1012df2ff75844c08fdacb8cdef652f7 2024-11-22T03:40:09,836 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/.tmp/B/6609b977b99c437babb9e14b5b5c0aa2 is 50, key is test_row_0/B:col10/1732246808240/Put/seqid=0 2024-11-22T03:40:09,846 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073741931_1107 (size=12301) 2024-11-22T03:40:09,891 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42545 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-22T03:40:09,891 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4. 2024-11-22T03:40:09,891 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4. as already flushing 2024-11-22T03:40:09,891 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4. 2024-11-22T03:40:09,891 ERROR [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => f970bd9f1e861e2b64a2fb87fb209ac4, NAME => 'TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:40:09,891 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => f970bd9f1e861e2b64a2fb87fb209ac4, NAME => 'TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:40:09,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42627 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f970bd9f1e861e2b64a2fb87fb209ac4, NAME => 'TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f970bd9f1e861e2b64a2fb87fb209ac4, NAME => 'TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:40:09,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-22T03:40:10,045 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42545 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-22T03:40:10,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4. 2024-11-22T03:40:10,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4. as already flushing 2024-11-22T03:40:10,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4. 2024-11-22T03:40:10,045 ERROR [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => f970bd9f1e861e2b64a2fb87fb209ac4, NAME => 'TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:40:10,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => f970bd9f1e861e2b64a2fb87fb209ac4, NAME => 'TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:40:10,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42627 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f970bd9f1e861e2b64a2fb87fb209ac4, NAME => 'TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f970bd9f1e861e2b64a2fb87fb209ac4, NAME => 'TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:40:10,052 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:10,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49612 deadline: 1732246870049, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:10,052 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:10,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49582 deadline: 1732246870049, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:10,053 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:10,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49566 deadline: 1732246870052, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:10,053 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:10,053 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:10,053 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:40:10,053 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:10,053 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:10,053 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:40:10,057 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:10,057 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:10,058 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:40:10,058 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:10,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49590 deadline: 1732246870057, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:10,058 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:10,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49606 deadline: 1732246870058, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:10,059 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:10,059 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:10,059 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:40:10,060 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:10,060 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:10,060 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:40:10,198 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42545 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-22T03:40:10,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4. 2024-11-22T03:40:10,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4. as already flushing 2024-11-22T03:40:10,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4. 2024-11-22T03:40:10,198 ERROR [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => f970bd9f1e861e2b64a2fb87fb209ac4, NAME => 'TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:40:10,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => f970bd9f1e861e2b64a2fb87fb209ac4, NAME => 'TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:40:10,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42627 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f970bd9f1e861e2b64a2fb87fb209ac4, NAME => 'TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f970bd9f1e861e2b64a2fb87fb209ac4, NAME => 'TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:40:10,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-22T03:40:10,247 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=454 (bloomFilter=true), to=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/.tmp/B/6609b977b99c437babb9e14b5b5c0aa2 2024-11-22T03:40:10,256 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/.tmp/C/459847ab448e480f929edc922aa3927c is 50, key is test_row_0/C:col10/1732246808240/Put/seqid=0 2024-11-22T03:40:10,261 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073741932_1108 (size=12301) 2024-11-22T03:40:10,351 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42545 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-22T03:40:10,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4. 2024-11-22T03:40:10,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4. as already flushing 2024-11-22T03:40:10,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4. 2024-11-22T03:40:10,352 ERROR [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => f970bd9f1e861e2b64a2fb87fb209ac4, NAME => 'TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:40:10,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => f970bd9f1e861e2b64a2fb87fb209ac4, NAME => 'TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:40:10,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42627 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f970bd9f1e861e2b64a2fb87fb209ac4, NAME => 'TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f970bd9f1e861e2b64a2fb87fb209ac4, NAME => 'TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:40:10,505 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42545 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-22T03:40:10,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4. 2024-11-22T03:40:10,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4. as already flushing 2024-11-22T03:40:10,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4. 2024-11-22T03:40:10,505 ERROR [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => f970bd9f1e861e2b64a2fb87fb209ac4, NAME => 'TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:40:10,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => f970bd9f1e861e2b64a2fb87fb209ac4, NAME => 'TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:40:10,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42627 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f970bd9f1e861e2b64a2fb87fb209ac4, NAME => 'TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f970bd9f1e861e2b64a2fb87fb209ac4, NAME => 'TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:40:10,560 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:10,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49582 deadline: 1732246870558, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:10,560 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:10,560 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:10,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49566 deadline: 1732246870558, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:10,560 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:10,560 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:40:10,561 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:10,561 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:10,561 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:40:10,573 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:10,573 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:10,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49612 deadline: 1732246870569, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:10,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49606 deadline: 1732246870569, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:10,574 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:10,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49590 deadline: 1732246870569, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:10,574 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:10,574 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:10,574 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:40:10,575 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:10,575 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:10,575 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:40:10,579 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:10,579 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:10,579 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:40:10,658 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42545 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-22T03:40:10,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4. 2024-11-22T03:40:10,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4. as already flushing 2024-11-22T03:40:10,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4. 2024-11-22T03:40:10,659 ERROR [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => f970bd9f1e861e2b64a2fb87fb209ac4, NAME => 'TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:40:10,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => f970bd9f1e861e2b64a2fb87fb209ac4, NAME => 'TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:40:10,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42627 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f970bd9f1e861e2b64a2fb87fb209ac4, NAME => 'TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f970bd9f1e861e2b64a2fb87fb209ac4, NAME => 'TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:40:10,662 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=454 (bloomFilter=true), to=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/.tmp/C/459847ab448e480f929edc922aa3927c 2024-11-22T03:40:10,670 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/.tmp/A/1012df2ff75844c08fdacb8cdef652f7 as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/A/1012df2ff75844c08fdacb8cdef652f7 2024-11-22T03:40:10,677 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/A/1012df2ff75844c08fdacb8cdef652f7, entries=250, sequenceid=454, filesize=16.8 K 2024-11-22T03:40:10,679 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/.tmp/B/6609b977b99c437babb9e14b5b5c0aa2 as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/B/6609b977b99c437babb9e14b5b5c0aa2 2024-11-22T03:40:10,700 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/B/6609b977b99c437babb9e14b5b5c0aa2, entries=150, sequenceid=454, filesize=12.0 K 2024-11-22T03:40:10,701 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/.tmp/C/459847ab448e480f929edc922aa3927c as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/C/459847ab448e480f929edc922aa3927c 2024-11-22T03:40:10,711 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/C/459847ab448e480f929edc922aa3927c, entries=150, sequenceid=454, filesize=12.0 K 2024-11-22T03:40:10,712 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=120.76 KB/123660 for f970bd9f1e861e2b64a2fb87fb209ac4 in 1322ms, sequenceid=454, compaction requested=true 2024-11-22T03:40:10,712 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f970bd9f1e861e2b64a2fb87fb209ac4: 2024-11-22T03:40:10,712 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f970bd9f1e861e2b64a2fb87fb209ac4:A, priority=-2147483648, current under compaction store size is 1 2024-11-22T03:40:10,712 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T03:40:10,712 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T03:40:10,712 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f970bd9f1e861e2b64a2fb87fb209ac4:B, priority=-2147483648, current under compaction store size is 2 2024-11-22T03:40:10,713 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T03:40:10,713 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T03:40:10,713 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f970bd9f1e861e2b64a2fb87fb209ac4:C, priority=-2147483648, current under compaction store size is 3 2024-11-22T03:40:10,713 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T03:40:10,714 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 42771 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T03:40:10,714 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37891 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T03:40:10,714 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.HStore(1541): f970bd9f1e861e2b64a2fb87fb209ac4/A is initiating minor compaction (all files) 2024-11-22T03:40:10,714 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HStore(1541): f970bd9f1e861e2b64a2fb87fb209ac4/B is initiating minor compaction (all files) 2024-11-22T03:40:10,714 INFO [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f970bd9f1e861e2b64a2fb87fb209ac4/B in TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4. 2024-11-22T03:40:10,714 INFO [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f970bd9f1e861e2b64a2fb87fb209ac4/A in TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4. 2024-11-22T03:40:10,714 INFO [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/B/3a5fb2435eae4c51ba4d913077c59c0e, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/B/85d6e91de066416c8b4b382e77605b84, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/B/6609b977b99c437babb9e14b5b5c0aa2] into tmpdir=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/.tmp, totalSize=37.0 K 2024-11-22T03:40:10,714 INFO [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/A/7409d20987cb4ff8b7bf232924241410, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/A/96968a6efacc4168b786ad9f0419e2d2, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/A/1012df2ff75844c08fdacb8cdef652f7] into tmpdir=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/.tmp, totalSize=41.8 K 2024-11-22T03:40:10,715 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] compactions.Compactor(225): Compacting 3a5fb2435eae4c51ba4d913077c59c0e, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=412, earliestPutTs=1732246807500 2024-11-22T03:40:10,715 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7409d20987cb4ff8b7bf232924241410, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=412, earliestPutTs=1732246807500 2024-11-22T03:40:10,716 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] compactions.Compactor(225): Compacting 85d6e91de066416c8b4b382e77605b84, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=435, earliestPutTs=1732246807859 2024-11-22T03:40:10,716 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] compactions.Compactor(225): Compacting 96968a6efacc4168b786ad9f0419e2d2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=435, earliestPutTs=1732246807859 2024-11-22T03:40:10,716 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] compactions.Compactor(225): Compacting 6609b977b99c437babb9e14b5b5c0aa2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=454, earliestPutTs=1732246808240 2024-11-22T03:40:10,717 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1012df2ff75844c08fdacb8cdef652f7, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=454, earliestPutTs=1732246808240 2024-11-22T03:40:10,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-22T03:40:10,733 INFO [RS:0;c85114ed5096:42545-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f970bd9f1e861e2b64a2fb87fb209ac4#B#compaction#96 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T03:40:10,734 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/.tmp/B/6a3f70e5f9a34c8ba27ce66caac8c258 is 50, key is test_row_0/B:col10/1732246808240/Put/seqid=0 2024-11-22T03:40:10,748 INFO [RS:0;c85114ed5096:42545-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f970bd9f1e861e2b64a2fb87fb209ac4#A#compaction#97 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T03:40:10,748 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/.tmp/A/4ec35656f37a4029acb763645fd5d7b6 is 50, key is test_row_0/A:col10/1732246808240/Put/seqid=0 2024-11-22T03:40:10,751 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073741933_1109 (size=13391) 2024-11-22T03:40:10,762 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/.tmp/B/6a3f70e5f9a34c8ba27ce66caac8c258 as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/B/6a3f70e5f9a34c8ba27ce66caac8c258 2024-11-22T03:40:10,771 INFO [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f970bd9f1e861e2b64a2fb87fb209ac4/B of f970bd9f1e861e2b64a2fb87fb209ac4 into 6a3f70e5f9a34c8ba27ce66caac8c258(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T03:40:10,771 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f970bd9f1e861e2b64a2fb87fb209ac4: 2024-11-22T03:40:10,771 INFO [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., storeName=f970bd9f1e861e2b64a2fb87fb209ac4/B, priority=13, startTime=1732246810712; duration=0sec 2024-11-22T03:40:10,771 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T03:40:10,771 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f970bd9f1e861e2b64a2fb87fb209ac4:B 2024-11-22T03:40:10,771 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T03:40:10,774 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37891 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T03:40:10,774 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HStore(1541): f970bd9f1e861e2b64a2fb87fb209ac4/C is initiating minor compaction (all files) 2024-11-22T03:40:10,774 INFO [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f970bd9f1e861e2b64a2fb87fb209ac4/C in TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4. 2024-11-22T03:40:10,774 INFO [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/C/7f2dbb2fab82448dabdfdcad1b3d7749, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/C/fe573883a1474be38ffd221a6ef3d57d, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/C/459847ab448e480f929edc922aa3927c] into tmpdir=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/.tmp, totalSize=37.0 K 2024-11-22T03:40:10,775 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] compactions.Compactor(225): Compacting 7f2dbb2fab82448dabdfdcad1b3d7749, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=412, earliestPutTs=1732246807500 2024-11-22T03:40:10,776 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] compactions.Compactor(225): Compacting fe573883a1474be38ffd221a6ef3d57d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=435, earliestPutTs=1732246807859 2024-11-22T03:40:10,776 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] compactions.Compactor(225): Compacting 459847ab448e480f929edc922aa3927c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=454, earliestPutTs=1732246808240 2024-11-22T03:40:10,803 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073741934_1110 (size=13391) 2024-11-22T03:40:10,810 INFO [RS:0;c85114ed5096:42545-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f970bd9f1e861e2b64a2fb87fb209ac4#C#compaction#98 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T03:40:10,811 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/.tmp/C/8656d12e45e248bc838b318639c7a673 is 50, key is test_row_0/C:col10/1732246808240/Put/seqid=0 2024-11-22T03:40:10,812 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42545 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-22T03:40:10,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4. 2024-11-22T03:40:10,813 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2902): Flushing f970bd9f1e861e2b64a2fb87fb209ac4 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2024-11-22T03:40:10,830 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/.tmp/A/c2e5994cc723423d8c9c71faee245392 is 50, key is test_row_0/A:col10/1732246809417/Put/seqid=0 2024-11-22T03:40:10,864 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073741936_1112 (size=12301) 2024-11-22T03:40:10,864 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=476 (bloomFilter=true), to=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/.tmp/A/c2e5994cc723423d8c9c71faee245392 2024-11-22T03:40:10,870 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073741935_1111 (size=13391) 2024-11-22T03:40:10,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/.tmp/B/47787cb3a6f64dfbae38f5ad071292b1 is 50, key is test_row_0/B:col10/1732246809417/Put/seqid=0 2024-11-22T03:40:10,884 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/.tmp/C/8656d12e45e248bc838b318639c7a673 as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/C/8656d12e45e248bc838b318639c7a673 2024-11-22T03:40:10,897 INFO [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f970bd9f1e861e2b64a2fb87fb209ac4/C of f970bd9f1e861e2b64a2fb87fb209ac4 into 8656d12e45e248bc838b318639c7a673(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T03:40:10,897 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f970bd9f1e861e2b64a2fb87fb209ac4: 2024-11-22T03:40:10,897 INFO [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., storeName=f970bd9f1e861e2b64a2fb87fb209ac4/C, priority=13, startTime=1732246810713; duration=0sec 2024-11-22T03:40:10,897 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T03:40:10,897 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f970bd9f1e861e2b64a2fb87fb209ac4:C 2024-11-22T03:40:10,909 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073741937_1113 (size=12301) 2024-11-22T03:40:10,914 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=476 (bloomFilter=true), to=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/.tmp/B/47787cb3a6f64dfbae38f5ad071292b1 2024-11-22T03:40:10,947 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/.tmp/C/8379710d47b946d2bafadda98cb2ed87 is 50, key is test_row_0/C:col10/1732246809417/Put/seqid=0 2024-11-22T03:40:10,976 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073741938_1114 (size=12301) 2024-11-22T03:40:10,977 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=476 (bloomFilter=true), to=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/.tmp/C/8379710d47b946d2bafadda98cb2ed87 2024-11-22T03:40:10,987 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/.tmp/A/c2e5994cc723423d8c9c71faee245392 as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/A/c2e5994cc723423d8c9c71faee245392 2024-11-22T03:40:10,996 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/A/c2e5994cc723423d8c9c71faee245392, entries=150, sequenceid=476, filesize=12.0 K 2024-11-22T03:40:10,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/.tmp/B/47787cb3a6f64dfbae38f5ad071292b1 as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/B/47787cb3a6f64dfbae38f5ad071292b1 2024-11-22T03:40:11,006 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/B/47787cb3a6f64dfbae38f5ad071292b1, entries=150, sequenceid=476, filesize=12.0 K 2024-11-22T03:40:11,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/.tmp/C/8379710d47b946d2bafadda98cb2ed87 as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/C/8379710d47b946d2bafadda98cb2ed87 2024-11-22T03:40:11,021 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/C/8379710d47b946d2bafadda98cb2ed87, entries=150, sequenceid=476, filesize=12.0 K 2024-11-22T03:40:11,023 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~317.11 KB/324720, currentSize=0 B/0 for f970bd9f1e861e2b64a2fb87fb209ac4 in 210ms, sequenceid=476, compaction requested=false 2024-11-22T03:40:11,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2603): Flush status journal for f970bd9f1e861e2b64a2fb87fb209ac4: 2024-11-22T03:40:11,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4. 2024-11-22T03:40:11,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=26 2024-11-22T03:40:11,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42627 {}] master.HMaster(4169): Remote procedure done, pid=26 2024-11-22T03:40:11,029 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=26, resume processing ppid=25 2024-11-22T03:40:11,029 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=26, ppid=25, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4410 sec 2024-11-22T03:40:11,039 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=25, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees in 1.4550 sec 2024-11-22T03:40:11,211 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/.tmp/A/4ec35656f37a4029acb763645fd5d7b6 as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/A/4ec35656f37a4029acb763645fd5d7b6 2024-11-22T03:40:11,219 INFO [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f970bd9f1e861e2b64a2fb87fb209ac4/A of f970bd9f1e861e2b64a2fb87fb209ac4 into 4ec35656f37a4029acb763645fd5d7b6(size=13.1 K), total size for store is 25.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T03:40:11,219 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f970bd9f1e861e2b64a2fb87fb209ac4: 2024-11-22T03:40:11,219 INFO [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., storeName=f970bd9f1e861e2b64a2fb87fb209ac4/A, priority=13, startTime=1732246810712; duration=0sec 2024-11-22T03:40:11,219 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T03:40:11,219 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f970bd9f1e861e2b64a2fb87fb209ac4:A 2024-11-22T03:40:11,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(8855): Flush requested on f970bd9f1e861e2b64a2fb87fb209ac4 2024-11-22T03:40:11,588 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-22T03:40:11,588 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f970bd9f1e861e2b64a2fb87fb209ac4 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-22T03:40:11,594 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/.tmp/A/60f4a58a80e149838132f3e12eccb350 is 50, key is test_row_0/A:col10/1732246811581/Put/seqid=0 2024-11-22T03:40:11,622 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073741939_1115 (size=12301) 2024-11-22T03:40:11,623 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=489 (bloomFilter=true), to=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/.tmp/A/60f4a58a80e149838132f3e12eccb350 2024-11-22T03:40:11,629 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:11,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49612 deadline: 1732246871623, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:11,630 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:11,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49606 deadline: 1732246871623, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:11,630 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:11,630 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:11,630 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:40:11,631 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:11,631 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:11,631 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:40:11,632 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:11,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49590 deadline: 1732246871626, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:11,632 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:11,633 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:11,633 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:11,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49582 deadline: 1732246871626, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:11,633 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:40:11,633 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:11,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49566 deadline: 1732246871628, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:11,633 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:11,633 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:11,634 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:40:11,634 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:11,634 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:11,634 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:40:11,642 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/.tmp/B/101e7d6ab06742c2b5c6fb7586a410ab is 50, key is test_row_0/B:col10/1732246811581/Put/seqid=0 2024-11-22T03:40:11,654 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073741940_1116 (size=12301) 2024-11-22T03:40:11,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-22T03:40:11,728 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-22T03:40:11,729 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42627 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-22T03:40:11,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42627 {}] procedure2.ProcedureExecutor(1139): Stored pid=27, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=27, table=TestAcidGuarantees 2024-11-22T03:40:11,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-22T03:40:11,731 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=27, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=27, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-22T03:40:11,731 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=27, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=27, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-22T03:40:11,732 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=28, ppid=27, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-22T03:40:11,739 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:11,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49612 deadline: 1732246871738, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:11,739 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:11,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49606 deadline: 1732246871738, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:11,739 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:11,739 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:11,739 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:11,739 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:11,739 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:40:11,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49590 deadline: 1732246871739, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:11,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49582 deadline: 1732246871739, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:11,740 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:11,740 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:11,740 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:11,740 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:11,740 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:11,740 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:11,740 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:40:11,740 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:40:11,740 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:11,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49566 deadline: 1732246871739, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:11,740 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:40:11,740 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:11,740 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:11,740 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:40:11,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-22T03:40:11,884 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42545 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=28 2024-11-22T03:40:11,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4. 2024-11-22T03:40:11,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4. as already flushing 2024-11-22T03:40:11,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4. 2024-11-22T03:40:11,885 ERROR [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] handler.RSProcedureHandler(58): pid=28 java.io.IOException: Unable to complete flush {ENCODED => f970bd9f1e861e2b64a2fb87fb209ac4, NAME => 'TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:40:11,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=28 java.io.IOException: Unable to complete flush {ENCODED => f970bd9f1e861e2b64a2fb87fb209ac4, NAME => 'TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:40:11,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42627 {}] master.HMaster(4177): Remote procedure failed, pid=28 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f970bd9f1e861e2b64a2fb87fb209ac4, NAME => 'TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f970bd9f1e861e2b64a2fb87fb209ac4, NAME => 'TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:40:11,950 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:11,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49612 deadline: 1732246871949, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:11,950 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:11,950 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:11,950 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:11,950 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:40:11,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49606 deadline: 1732246871949, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:11,951 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:11,951 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:11,951 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:11,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49582 deadline: 1732246871949, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:11,951 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:40:11,951 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:11,951 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:11,951 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:11,951 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:40:11,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49590 deadline: 1732246871949, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:11,951 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:11,951 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:11,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49566 deadline: 1732246871949, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:11,951 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:11,952 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:40:11,952 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:11,952 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:11,952 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:40:12,037 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42545 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=28 2024-11-22T03:40:12,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4. 2024-11-22T03:40:12,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4. as already flushing 2024-11-22T03:40:12,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4. 2024-11-22T03:40:12,038 ERROR [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] handler.RSProcedureHandler(58): pid=28 java.io.IOException: Unable to complete flush {ENCODED => f970bd9f1e861e2b64a2fb87fb209ac4, NAME => 'TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:40:12,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=28 java.io.IOException: Unable to complete flush {ENCODED => f970bd9f1e861e2b64a2fb87fb209ac4, NAME => 'TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:40:12,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42627 {}] master.HMaster(4177): Remote procedure failed, pid=28 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f970bd9f1e861e2b64a2fb87fb209ac4, NAME => 'TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f970bd9f1e861e2b64a2fb87fb209ac4, NAME => 'TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:40:12,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-22T03:40:12,055 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=489 (bloomFilter=true), to=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/.tmp/B/101e7d6ab06742c2b5c6fb7586a410ab 2024-11-22T03:40:12,063 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/.tmp/C/b11298ed22f943ed8cd400e2ac8fc876 is 50, key is test_row_0/C:col10/1732246811581/Put/seqid=0 2024-11-22T03:40:12,067 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073741941_1117 (size=12301) 2024-11-22T03:40:12,190 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42545 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=28 2024-11-22T03:40:12,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4. 2024-11-22T03:40:12,191 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4. as already flushing 2024-11-22T03:40:12,191 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4. 2024-11-22T03:40:12,191 ERROR [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] handler.RSProcedureHandler(58): pid=28 java.io.IOException: Unable to complete flush {ENCODED => f970bd9f1e861e2b64a2fb87fb209ac4, NAME => 'TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:40:12,191 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=28 java.io.IOException: Unable to complete flush {ENCODED => f970bd9f1e861e2b64a2fb87fb209ac4, NAME => 'TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:40:12,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42627 {}] master.HMaster(4177): Remote procedure failed, pid=28 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f970bd9f1e861e2b64a2fb87fb209ac4, NAME => 'TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f970bd9f1e861e2b64a2fb87fb209ac4, NAME => 'TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:40:12,259 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:12,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49612 deadline: 1732246872258, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:12,259 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:12,259 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:12,259 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:12,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49606 deadline: 1732246872259, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:12,259 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:40:12,260 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:12,260 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:12,260 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:40:12,264 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:12,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49582 deadline: 1732246872259, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:12,264 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:12,265 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:12,265 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:12,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49590 deadline: 1732246872259, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:12,265 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:40:12,265 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:12,265 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:12,265 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:12,265 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:40:12,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49566 deadline: 1732246872259, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:12,265 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:12,265 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:12,265 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:40:12,343 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42545 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=28 2024-11-22T03:40:12,344 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4. 2024-11-22T03:40:12,344 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4. as already flushing 2024-11-22T03:40:12,344 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4. 2024-11-22T03:40:12,344 ERROR [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] handler.RSProcedureHandler(58): pid=28 java.io.IOException: Unable to complete flush {ENCODED => f970bd9f1e861e2b64a2fb87fb209ac4, NAME => 'TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:40:12,344 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=28 java.io.IOException: Unable to complete flush {ENCODED => f970bd9f1e861e2b64a2fb87fb209ac4, NAME => 'TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:40:12,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42627 {}] master.HMaster(4177): Remote procedure failed, pid=28 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f970bd9f1e861e2b64a2fb87fb209ac4, NAME => 'TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f970bd9f1e861e2b64a2fb87fb209ac4, NAME => 'TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:40:12,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-22T03:40:12,467 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=489 (bloomFilter=true), to=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/.tmp/C/b11298ed22f943ed8cd400e2ac8fc876 2024-11-22T03:40:12,474 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/.tmp/A/60f4a58a80e149838132f3e12eccb350 as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/A/60f4a58a80e149838132f3e12eccb350 2024-11-22T03:40:12,480 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/A/60f4a58a80e149838132f3e12eccb350, entries=150, sequenceid=489, filesize=12.0 K 2024-11-22T03:40:12,481 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/.tmp/B/101e7d6ab06742c2b5c6fb7586a410ab as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/B/101e7d6ab06742c2b5c6fb7586a410ab 2024-11-22T03:40:12,489 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/B/101e7d6ab06742c2b5c6fb7586a410ab, entries=150, sequenceid=489, filesize=12.0 K 2024-11-22T03:40:12,491 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/.tmp/C/b11298ed22f943ed8cd400e2ac8fc876 as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/C/b11298ed22f943ed8cd400e2ac8fc876 2024-11-22T03:40:12,496 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/C/b11298ed22f943ed8cd400e2ac8fc876, entries=150, sequenceid=489, filesize=12.0 K 2024-11-22T03:40:12,497 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42545 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=28 2024-11-22T03:40:12,497 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for f970bd9f1e861e2b64a2fb87fb209ac4 in 909ms, sequenceid=489, compaction requested=true 2024-11-22T03:40:12,497 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f970bd9f1e861e2b64a2fb87fb209ac4: 2024-11-22T03:40:12,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4. 2024-11-22T03:40:12,497 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(2902): Flushing f970bd9f1e861e2b64a2fb87fb209ac4 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-22T03:40:12,497 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T03:40:12,497 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f970bd9f1e861e2b64a2fb87fb209ac4:A, priority=-2147483648, current under compaction store size is 1 2024-11-22T03:40:12,497 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T03:40:12,498 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f970bd9f1e861e2b64a2fb87fb209ac4:B, priority=-2147483648, current under compaction store size is 2 2024-11-22T03:40:12,498 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T03:40:12,498 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T03:40:12,498 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f970bd9f1e861e2b64a2fb87fb209ac4:C, priority=-2147483648, current under compaction store size is 3 2024-11-22T03:40:12,498 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T03:40:12,502 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37993 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T03:40:12,503 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.HStore(1541): f970bd9f1e861e2b64a2fb87fb209ac4/A is initiating minor compaction (all files) 2024-11-22T03:40:12,503 INFO [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f970bd9f1e861e2b64a2fb87fb209ac4/A in TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4. 2024-11-22T03:40:12,503 INFO [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/A/4ec35656f37a4029acb763645fd5d7b6, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/A/c2e5994cc723423d8c9c71faee245392, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/A/60f4a58a80e149838132f3e12eccb350] into tmpdir=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/.tmp, totalSize=37.1 K 2024-11-22T03:40:12,503 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4ec35656f37a4029acb763645fd5d7b6, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=454, earliestPutTs=1732246808240 2024-11-22T03:40:12,503 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37993 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T03:40:12,504 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HStore(1541): f970bd9f1e861e2b64a2fb87fb209ac4/B is initiating minor compaction (all files) 2024-11-22T03:40:12,504 INFO [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f970bd9f1e861e2b64a2fb87fb209ac4/B in TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4. 2024-11-22T03:40:12,504 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] compactions.Compactor(225): Compacting c2e5994cc723423d8c9c71faee245392, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=476, earliestPutTs=1732246809414 2024-11-22T03:40:12,504 INFO [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/B/6a3f70e5f9a34c8ba27ce66caac8c258, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/B/47787cb3a6f64dfbae38f5ad071292b1, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/B/101e7d6ab06742c2b5c6fb7586a410ab] into tmpdir=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/.tmp, totalSize=37.1 K 2024-11-22T03:40:12,505 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] compactions.Compactor(225): Compacting 6a3f70e5f9a34c8ba27ce66caac8c258, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=454, earliestPutTs=1732246808240 2024-11-22T03:40:12,505 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] compactions.Compactor(225): Compacting 60f4a58a80e149838132f3e12eccb350, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=489, earliestPutTs=1732246811577 2024-11-22T03:40:12,506 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] compactions.Compactor(225): Compacting 47787cb3a6f64dfbae38f5ad071292b1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=476, earliestPutTs=1732246809414 2024-11-22T03:40:12,506 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] compactions.Compactor(225): Compacting 101e7d6ab06742c2b5c6fb7586a410ab, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=489, earliestPutTs=1732246811577 2024-11-22T03:40:12,520 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/.tmp/A/497f5b96b7d14648a6380d0146a6b40d is 50, key is test_row_0/A:col10/1732246811621/Put/seqid=0 2024-11-22T03:40:12,521 INFO [RS:0;c85114ed5096:42545-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f970bd9f1e861e2b64a2fb87fb209ac4#A#compaction#106 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T03:40:12,522 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/.tmp/A/5ef48e3dd17d48568d9244f882b240a6 is 50, key is test_row_0/A:col10/1732246811581/Put/seqid=0 2024-11-22T03:40:12,524 INFO [RS:0;c85114ed5096:42545-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f970bd9f1e861e2b64a2fb87fb209ac4#B#compaction#107 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T03:40:12,524 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/.tmp/B/20cda7031cd546f9a7d2f73ae220bada is 50, key is test_row_0/B:col10/1732246811581/Put/seqid=0 2024-11-22T03:40:12,535 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073741943_1119 (size=13493) 2024-11-22T03:40:12,537 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073741942_1118 (size=12301) 2024-11-22T03:40:12,537 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=514 (bloomFilter=true), to=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/.tmp/A/497f5b96b7d14648a6380d0146a6b40d 2024-11-22T03:40:12,540 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073741944_1120 (size=13493) 2024-11-22T03:40:12,546 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/.tmp/A/5ef48e3dd17d48568d9244f882b240a6 as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/A/5ef48e3dd17d48568d9244f882b240a6 2024-11-22T03:40:12,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/.tmp/B/774c2a8dd66146aa960f86b2ee71ad2c is 50, key is test_row_0/B:col10/1732246811621/Put/seqid=0 2024-11-22T03:40:12,553 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/.tmp/B/20cda7031cd546f9a7d2f73ae220bada as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/B/20cda7031cd546f9a7d2f73ae220bada 2024-11-22T03:40:12,556 INFO [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f970bd9f1e861e2b64a2fb87fb209ac4/A of f970bd9f1e861e2b64a2fb87fb209ac4 into 5ef48e3dd17d48568d9244f882b240a6(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T03:40:12,556 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f970bd9f1e861e2b64a2fb87fb209ac4: 2024-11-22T03:40:12,556 INFO [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., storeName=f970bd9f1e861e2b64a2fb87fb209ac4/A, priority=13, startTime=1732246812497; duration=0sec 2024-11-22T03:40:12,556 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T03:40:12,557 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f970bd9f1e861e2b64a2fb87fb209ac4:A 2024-11-22T03:40:12,557 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T03:40:12,558 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37993 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T03:40:12,558 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.HStore(1541): f970bd9f1e861e2b64a2fb87fb209ac4/C is initiating minor compaction (all files) 2024-11-22T03:40:12,559 INFO [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f970bd9f1e861e2b64a2fb87fb209ac4/C in TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4. 2024-11-22T03:40:12,559 INFO [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/C/8656d12e45e248bc838b318639c7a673, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/C/8379710d47b946d2bafadda98cb2ed87, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/C/b11298ed22f943ed8cd400e2ac8fc876] into tmpdir=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/.tmp, totalSize=37.1 K 2024-11-22T03:40:12,559 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8656d12e45e248bc838b318639c7a673, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=454, earliestPutTs=1732246808240 2024-11-22T03:40:12,560 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8379710d47b946d2bafadda98cb2ed87, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=476, earliestPutTs=1732246809414 2024-11-22T03:40:12,560 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] compactions.Compactor(225): Compacting b11298ed22f943ed8cd400e2ac8fc876, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=489, earliestPutTs=1732246811577 2024-11-22T03:40:12,562 INFO [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f970bd9f1e861e2b64a2fb87fb209ac4/B of f970bd9f1e861e2b64a2fb87fb209ac4 into 20cda7031cd546f9a7d2f73ae220bada(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T03:40:12,562 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f970bd9f1e861e2b64a2fb87fb209ac4: 2024-11-22T03:40:12,562 INFO [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., storeName=f970bd9f1e861e2b64a2fb87fb209ac4/B, priority=13, startTime=1732246812497; duration=0sec 2024-11-22T03:40:12,562 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T03:40:12,562 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f970bd9f1e861e2b64a2fb87fb209ac4:B 2024-11-22T03:40:12,571 INFO [RS:0;c85114ed5096:42545-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f970bd9f1e861e2b64a2fb87fb209ac4#C#compaction#109 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T03:40:12,571 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/.tmp/C/b71215a78e0844cbb6df1a176d4fcfcc is 50, key is test_row_0/C:col10/1732246811581/Put/seqid=0 2024-11-22T03:40:12,584 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073741945_1121 (size=12301) 2024-11-22T03:40:12,594 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073741946_1122 (size=13493) 2024-11-22T03:40:12,607 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/.tmp/C/b71215a78e0844cbb6df1a176d4fcfcc as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/C/b71215a78e0844cbb6df1a176d4fcfcc 2024-11-22T03:40:12,616 INFO [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f970bd9f1e861e2b64a2fb87fb209ac4/C of f970bd9f1e861e2b64a2fb87fb209ac4 into b71215a78e0844cbb6df1a176d4fcfcc(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T03:40:12,616 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f970bd9f1e861e2b64a2fb87fb209ac4: 2024-11-22T03:40:12,616 INFO [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., storeName=f970bd9f1e861e2b64a2fb87fb209ac4/C, priority=13, startTime=1732246812498; duration=0sec 2024-11-22T03:40:12,616 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T03:40:12,616 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f970bd9f1e861e2b64a2fb87fb209ac4:C 2024-11-22T03:40:12,771 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4. as already flushing 2024-11-22T03:40:12,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(8855): Flush requested on f970bd9f1e861e2b64a2fb87fb209ac4 2024-11-22T03:40:12,780 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:12,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49612 deadline: 1732246872775, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:12,780 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:12,781 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:12,780 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:12,781 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:12,781 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:40:12,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49582 deadline: 1732246872779, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:12,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49606 deadline: 1732246872779, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:12,781 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:12,781 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:12,781 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:12,781 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:12,781 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:40:12,781 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:12,781 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:40:12,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49566 deadline: 1732246872780, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:12,782 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:12,782 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:12,782 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:40:12,782 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:12,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49590 deadline: 1732246872779, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:12,782 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:12,782 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:12,782 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:40:12,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-22T03:40:12,890 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:12,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49612 deadline: 1732246872889, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:12,891 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:12,891 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:12,892 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:40:12,892 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:12,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49606 deadline: 1732246872889, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:12,893 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:12,893 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:12,893 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:40:12,893 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:12,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49566 deadline: 1732246872890, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:12,894 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:12,894 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:12,894 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:40:12,987 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=514 (bloomFilter=true), to=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/.tmp/B/774c2a8dd66146aa960f86b2ee71ad2c 2024-11-22T03:40:13,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/.tmp/C/57217b1e38824a8fb207307d5ed9baa1 is 50, key is test_row_0/C:col10/1732246811621/Put/seqid=0 2024-11-22T03:40:13,077 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073741947_1123 (size=12301) 2024-11-22T03:40:13,078 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=514 (bloomFilter=true), to=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/.tmp/C/57217b1e38824a8fb207307d5ed9baa1 2024-11-22T03:40:13,092 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/.tmp/A/497f5b96b7d14648a6380d0146a6b40d as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/A/497f5b96b7d14648a6380d0146a6b40d 2024-11-22T03:40:13,102 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/A/497f5b96b7d14648a6380d0146a6b40d, entries=150, sequenceid=514, filesize=12.0 K 2024-11-22T03:40:13,103 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:13,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49606 deadline: 1732246873099, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:13,103 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:13,103 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:13,103 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:13,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49612 deadline: 1732246873100, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:13,104 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:40:13,104 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:13,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49566 deadline: 1732246873100, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:13,106 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:13,106 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:13,107 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:40:13,107 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:13,107 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:13,107 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:40:13,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/.tmp/B/774c2a8dd66146aa960f86b2ee71ad2c as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/B/774c2a8dd66146aa960f86b2ee71ad2c 2024-11-22T03:40:13,119 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/B/774c2a8dd66146aa960f86b2ee71ad2c, entries=150, sequenceid=514, filesize=12.0 K 2024-11-22T03:40:13,123 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/.tmp/C/57217b1e38824a8fb207307d5ed9baa1 as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/C/57217b1e38824a8fb207307d5ed9baa1 2024-11-22T03:40:13,142 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/C/57217b1e38824a8fb207307d5ed9baa1, entries=150, sequenceid=514, filesize=12.0 K 2024-11-22T03:40:13,144 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for f970bd9f1e861e2b64a2fb87fb209ac4 in 646ms, sequenceid=514, compaction requested=false 2024-11-22T03:40:13,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(2603): Flush status journal for f970bd9f1e861e2b64a2fb87fb209ac4: 2024-11-22T03:40:13,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4. 2024-11-22T03:40:13,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=28 2024-11-22T03:40:13,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42627 {}] master.HMaster(4169): Remote procedure done, pid=28 2024-11-22T03:40:13,148 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=28, resume processing ppid=27 2024-11-22T03:40:13,149 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=28, ppid=27, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4140 sec 2024-11-22T03:40:13,154 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=27, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=27, table=TestAcidGuarantees in 1.4220 sec 2024-11-22T03:40:13,225 INFO [Thread-158 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-158. 2024-11-22T03:40:13,226 DEBUG [Thread-158 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-22T03:40:13,229 INFO [Thread-156 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-156. 2024-11-22T03:40:13,229 INFO [Thread-154 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-154. 2024-11-22T03:40:13,229 DEBUG [Thread-154 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-22T03:40:13,229 DEBUG [Thread-156 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-22T03:40:13,231 INFO [Thread-160 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-160. 2024-11-22T03:40:13,231 DEBUG [Thread-160 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-22T03:40:13,233 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T03:40:13,233 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T03:40:13,233 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T03:40:13,234 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T03:40:13,234 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-22T03:40:13,234 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-22T03:40:13,234 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T03:40:13,234 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T03:40:13,234 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T03:40:13,234 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-22T03:40:13,235 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T03:40:13,235 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-22T03:40:13,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(8855): Flush requested on f970bd9f1e861e2b64a2fb87fb209ac4 2024-11-22T03:40:13,410 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-22T03:40:13,410 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f970bd9f1e861e2b64a2fb87fb209ac4 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-22T03:40:13,411 INFO [Thread-145 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-145. 2024-11-22T03:40:13,411 DEBUG [Thread-145 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-22T03:40:13,411 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T03:40:13,411 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T03:40:13,411 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-22T03:40:13,415 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/.tmp/A/fbe68947faba4993a94d066c07104d6d is 50, key is test_row_0/A:col10/1732246812773/Put/seqid=0 2024-11-22T03:40:13,419 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073741948_1124 (size=12301) 2024-11-22T03:40:13,420 INFO [Thread-149 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-149. 2024-11-22T03:40:13,420 DEBUG [Thread-149 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-22T03:40:13,420 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T03:40:13,420 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T03:40:13,420 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-22T03:40:13,421 INFO [Thread-147 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-147. 2024-11-22T03:40:13,421 DEBUG [Thread-147 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-22T03:40:13,421 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T03:40:13,421 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T03:40:13,421 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-22T03:40:13,795 INFO [Thread-143 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-143. 2024-11-22T03:40:13,795 INFO [Thread-151 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-151. 2024-11-22T03:40:13,795 DEBUG [Thread-143 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-22T03:40:13,795 DEBUG [Thread-151 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-22T03:40:13,795 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T03:40:13,795 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T03:40:13,796 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T03:40:13,796 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T03:40:13,796 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-22T03:40:13,796 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-22T03:40:13,820 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=529 (bloomFilter=true), to=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/.tmp/A/fbe68947faba4993a94d066c07104d6d 2024-11-22T03:40:13,827 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/.tmp/B/15aef831f06b41048e24feb572503cf6 is 50, key is test_row_0/B:col10/1732246812773/Put/seqid=0 2024-11-22T03:40:13,831 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073741949_1125 (size=12301) 2024-11-22T03:40:13,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-22T03:40:13,880 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-22T03:40:13,880 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-22T03:40:13,880 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 86 2024-11-22T03:40:13,880 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 85 2024-11-22T03:40:13,880 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 90 2024-11-22T03:40:13,880 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 78 2024-11-22T03:40:13,880 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 79 2024-11-22T03:40:13,880 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-22T03:40:13,880 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3146 2024-11-22T03:40:13,880 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3108 2024-11-22T03:40:13,880 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-22T03:40:13,880 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1484 2024-11-22T03:40:13,881 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4449 rows 2024-11-22T03:40:13,881 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1457 2024-11-22T03:40:13,881 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4370 rows 2024-11-22T03:40:13,881 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-22T03:40:13,881 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMixedAtomicity(AcidGuaranteesTestBase.java:117) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-22T03:40:13,881 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T03:40:13,881 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T03:40:13,881 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-22T03:40:13,888 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42627 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-22T03:40:13,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42627 {}] procedure2.ProcedureExecutor(1139): Stored pid=29, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-22T03:40:13,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-22T03:40:13,897 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732246813897"}]},"ts":"1732246813897"} 2024-11-22T03:40:13,900 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-22T03:40:13,900 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-22T03:40:13,902 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=30, ppid=29, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-22T03:40:13,906 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f970bd9f1e861e2b64a2fb87fb209ac4, UNASSIGN}] 2024-11-22T03:40:13,907 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f970bd9f1e861e2b64a2fb87fb209ac4, UNASSIGN 2024-11-22T03:40:13,909 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=31 updating hbase:meta row=f970bd9f1e861e2b64a2fb87fb209ac4, regionState=CLOSING, regionLocation=c85114ed5096,42545,1732246786482 2024-11-22T03:40:13,911 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=f970bd9f1e861e2b64a2fb87fb209ac4, UNASSIGN because future has completed 2024-11-22T03:40:13,912 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-22T03:40:13,912 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42627 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=CLOSING, location=c85114ed5096,42545,1732246786482, table=TestAcidGuarantees, region=f970bd9f1e861e2b64a2fb87fb209ac4. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-22T03:40:13,912 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=32, ppid=31, state=RUNNABLE, hasLock=false; CloseRegionProcedure f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482}] 2024-11-22T03:40:14,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-22T03:40:14,068 INFO [RS_CLOSE_REGION-regionserver/c85114ed5096:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] handler.UnassignRegionHandler(122): Close f970bd9f1e861e2b64a2fb87fb209ac4 2024-11-22T03:40:14,068 DEBUG [RS_CLOSE_REGION-regionserver/c85114ed5096:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-22T03:40:14,069 DEBUG [RS_CLOSE_REGION-regionserver/c85114ed5096:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1722): Closing f970bd9f1e861e2b64a2fb87fb209ac4, disabling compactions & flushes 2024-11-22T03:40:14,069 DEBUG [RS_CLOSE_REGION-regionserver/c85114ed5096:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1993): waiting for 0 compactions & cache flush to complete for region TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4. 2024-11-22T03:40:14,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-22T03:40:14,233 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=529 (bloomFilter=true), to=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/.tmp/B/15aef831f06b41048e24feb572503cf6 2024-11-22T03:40:14,247 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/.tmp/C/2608aa120f7b4c91a5318ac30f6f7fce is 50, key is test_row_0/C:col10/1732246812773/Put/seqid=0 2024-11-22T03:40:14,251 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073741950_1126 (size=12301) 2024-11-22T03:40:14,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-22T03:40:14,652 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=529 (bloomFilter=true), to=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/.tmp/C/2608aa120f7b4c91a5318ac30f6f7fce 2024-11-22T03:40:14,663 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/.tmp/A/fbe68947faba4993a94d066c07104d6d as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/A/fbe68947faba4993a94d066c07104d6d 2024-11-22T03:40:14,671 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/A/fbe68947faba4993a94d066c07104d6d, entries=150, sequenceid=529, filesize=12.0 K 2024-11-22T03:40:14,673 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/.tmp/B/15aef831f06b41048e24feb572503cf6 as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/B/15aef831f06b41048e24feb572503cf6 2024-11-22T03:40:14,679 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/B/15aef831f06b41048e24feb572503cf6, entries=150, sequenceid=529, filesize=12.0 K 2024-11-22T03:40:14,680 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/.tmp/C/2608aa120f7b4c91a5318ac30f6f7fce as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/C/2608aa120f7b4c91a5318ac30f6f7fce 2024-11-22T03:40:14,684 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/C/2608aa120f7b4c91a5318ac30f6f7fce, entries=150, sequenceid=529, filesize=12.0 K 2024-11-22T03:40:14,685 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=26.84 KB/27480 for f970bd9f1e861e2b64a2fb87fb209ac4 in 1275ms, sequenceid=529, compaction requested=true 2024-11-22T03:40:14,685 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f970bd9f1e861e2b64a2fb87fb209ac4: 2024-11-22T03:40:14,685 INFO [RS_CLOSE_REGION-regionserver/c85114ed5096:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4. 2024-11-22T03:40:14,685 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f970bd9f1e861e2b64a2fb87fb209ac4:A, priority=-2147483648, current under compaction store size is 1 2024-11-22T03:40:14,685 DEBUG [RS_CLOSE_REGION-regionserver/c85114ed5096:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4. 2024-11-22T03:40:14,685 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T03:40:14,685 DEBUG [RS_CLOSE_REGION-regionserver/c85114ed5096:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4. after waiting 0 ms 2024-11-22T03:40:14,685 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f970bd9f1e861e2b64a2fb87fb209ac4:B, priority=-2147483648, current under compaction store size is 2 2024-11-22T03:40:14,685 DEBUG [RS_CLOSE_REGION-regionserver/c85114ed5096:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4. 2024-11-22T03:40:14,685 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T03:40:14,685 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f970bd9f1e861e2b64a2fb87fb209ac4:C, priority=-2147483648, current under compaction store size is 3 2024-11-22T03:40:14,685 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4. because compaction request was cancelled 2024-11-22T03:40:14,685 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4. because compaction request was cancelled 2024-11-22T03:40:14,685 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T03:40:14,685 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f970bd9f1e861e2b64a2fb87fb209ac4:B 2024-11-22T03:40:14,685 INFO [RS_CLOSE_REGION-regionserver/c85114ed5096:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(2902): Flushing f970bd9f1e861e2b64a2fb87fb209ac4 3/3 column families, dataSize=26.84 KB heapSize=71.06 KB 2024-11-22T03:40:14,685 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f970bd9f1e861e2b64a2fb87fb209ac4:A 2024-11-22T03:40:14,685 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4. because compaction request was cancelled 2024-11-22T03:40:14,686 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f970bd9f1e861e2b64a2fb87fb209ac4:C 2024-11-22T03:40:14,690 DEBUG [RS_CLOSE_REGION-regionserver/c85114ed5096:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/.tmp/A/c24c7f9ffe744e84b219368ab3e57429 is 50, key is test_row_2/A:col10/1732246813791/Put/seqid=0 2024-11-22T03:40:14,694 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073741951_1127 (size=7415) 2024-11-22T03:40:14,822 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-22T03:40:15,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-22T03:40:15,094 INFO [RS_CLOSE_REGION-regionserver/c85114ed5096:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=536 (bloomFilter=true), to=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/.tmp/A/c24c7f9ffe744e84b219368ab3e57429 2024-11-22T03:40:15,103 DEBUG [RS_CLOSE_REGION-regionserver/c85114ed5096:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/.tmp/B/9f70bddec72c4126b2cf014172f254c4 is 50, key is test_row_2/B:col10/1732246813791/Put/seqid=0 2024-11-22T03:40:15,108 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073741952_1128 (size=7415) 2024-11-22T03:40:15,509 INFO [RS_CLOSE_REGION-regionserver/c85114ed5096:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=536 (bloomFilter=true), to=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/.tmp/B/9f70bddec72c4126b2cf014172f254c4 2024-11-22T03:40:15,517 DEBUG [RS_CLOSE_REGION-regionserver/c85114ed5096:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/.tmp/C/cc39764ab6e54cf39cfc8fd0bd0dc4f2 is 50, key is test_row_2/C:col10/1732246813791/Put/seqid=0 2024-11-22T03:40:15,521 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073741953_1129 (size=7415) 2024-11-22T03:40:15,922 INFO [RS_CLOSE_REGION-regionserver/c85114ed5096:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=536 (bloomFilter=true), to=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/.tmp/C/cc39764ab6e54cf39cfc8fd0bd0dc4f2 2024-11-22T03:40:15,928 DEBUG [RS_CLOSE_REGION-regionserver/c85114ed5096:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/.tmp/A/c24c7f9ffe744e84b219368ab3e57429 as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/A/c24c7f9ffe744e84b219368ab3e57429 2024-11-22T03:40:15,938 INFO [RS_CLOSE_REGION-regionserver/c85114ed5096:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/A/c24c7f9ffe744e84b219368ab3e57429, entries=50, sequenceid=536, filesize=7.2 K 2024-11-22T03:40:15,940 DEBUG [RS_CLOSE_REGION-regionserver/c85114ed5096:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/.tmp/B/9f70bddec72c4126b2cf014172f254c4 as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/B/9f70bddec72c4126b2cf014172f254c4 2024-11-22T03:40:15,949 INFO [RS_CLOSE_REGION-regionserver/c85114ed5096:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/B/9f70bddec72c4126b2cf014172f254c4, entries=50, sequenceid=536, filesize=7.2 K 2024-11-22T03:40:15,950 DEBUG [RS_CLOSE_REGION-regionserver/c85114ed5096:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/.tmp/C/cc39764ab6e54cf39cfc8fd0bd0dc4f2 as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/C/cc39764ab6e54cf39cfc8fd0bd0dc4f2 2024-11-22T03:40:15,955 INFO [RS_CLOSE_REGION-regionserver/c85114ed5096:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/C/cc39764ab6e54cf39cfc8fd0bd0dc4f2, entries=50, sequenceid=536, filesize=7.2 K 2024-11-22T03:40:15,956 INFO [RS_CLOSE_REGION-regionserver/c85114ed5096:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~71.02 KB/72720, currentSize=0 B/0 for f970bd9f1e861e2b64a2fb87fb209ac4 in 1271ms, sequenceid=536, compaction requested=true 2024-11-22T03:40:15,957 DEBUG [StoreCloser-TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/A/7707745419cb47b79b1297ab3f9881bb, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/A/cd4945c96adf4832a8d7671f9f41d297, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/A/8ce92063167a442187a26900d790f5e3, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/A/cfb481a93dfa49358f7f828d17d57846, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/A/2e92634f8b1449f0aa5485b493ef90e7, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/A/dc9bd375e9c24a4b959e574cba27aac0, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/A/05c1c9a8c4a54140be86638ea9b9e584, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/A/b05283494a0c417baf99a6549ef6981d, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/A/20e630d42fc848cd9efb0e5638edf4d8, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/A/0a82e43cb2cb41838b61e97b23444822, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/A/36f4ff699dd84da1913edc79545b1f8f, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/A/f7a8dc7598c0453695f04d4f4bdb1c5d, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/A/05035c1f745345c3b99912723bf86b20, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/A/00f669db6b0b4f248e23aea844f17710, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/A/79873c10df23470e91a8370dd7307ef0, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/A/ab5718b4e15340b08839c04e41e389eb, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/A/c2cfd0728eb0486ebd70c72ce57458ac, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/A/d89aa080a04b44c6b46fa211652eee8f, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/A/a0cec3316e6c4395ab8f80f15ed72fee, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/A/32714810e79943e489b8965490a9deb0, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/A/2a450a145c3044f69195ccd86386ed94, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/A/daebc7c44ab644e79e0306a701ccae2c, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/A/d405092cdcbd408caf5f8951e21805c3, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/A/d6f08aef05934343b1f738a5442203b6, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/A/c0bd9fec57124d94b2204832a0dea86b, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/A/5818145b02e14d9f9c0b2427280fb481, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/A/8102c612cad44458827ed502b2e65903, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/A/19d5c76c52cb46789f6bf5885b7650e4, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/A/346aa3a7f4c340d2bd5f2f50473fec1d, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/A/7409d20987cb4ff8b7bf232924241410, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/A/96968a6efacc4168b786ad9f0419e2d2, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/A/1012df2ff75844c08fdacb8cdef652f7, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/A/4ec35656f37a4029acb763645fd5d7b6, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/A/c2e5994cc723423d8c9c71faee245392, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/A/60f4a58a80e149838132f3e12eccb350] to archive 2024-11-22T03:40:15,960 DEBUG [StoreCloser-TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-22T03:40:15,968 DEBUG [StoreCloser-TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/A/7707745419cb47b79b1297ab3f9881bb to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/A/7707745419cb47b79b1297ab3f9881bb 2024-11-22T03:40:15,971 DEBUG [StoreCloser-TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/A/cd4945c96adf4832a8d7671f9f41d297 to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/A/cd4945c96adf4832a8d7671f9f41d297 2024-11-22T03:40:15,976 DEBUG [StoreCloser-TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/A/8ce92063167a442187a26900d790f5e3 to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/A/8ce92063167a442187a26900d790f5e3 2024-11-22T03:40:15,979 DEBUG [StoreCloser-TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/A/cfb481a93dfa49358f7f828d17d57846 to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/A/cfb481a93dfa49358f7f828d17d57846 2024-11-22T03:40:15,982 DEBUG [StoreCloser-TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/A/2e92634f8b1449f0aa5485b493ef90e7 to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/A/2e92634f8b1449f0aa5485b493ef90e7 2024-11-22T03:40:15,985 DEBUG [StoreCloser-TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/A/dc9bd375e9c24a4b959e574cba27aac0 to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/A/dc9bd375e9c24a4b959e574cba27aac0 2024-11-22T03:40:15,987 DEBUG [StoreCloser-TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/A/05c1c9a8c4a54140be86638ea9b9e584 to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/A/05c1c9a8c4a54140be86638ea9b9e584 2024-11-22T03:40:15,990 DEBUG [StoreCloser-TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/A/b05283494a0c417baf99a6549ef6981d to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/A/b05283494a0c417baf99a6549ef6981d 2024-11-22T03:40:15,992 DEBUG [StoreCloser-TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/A/20e630d42fc848cd9efb0e5638edf4d8 to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/A/20e630d42fc848cd9efb0e5638edf4d8 2024-11-22T03:40:15,998 DEBUG [StoreCloser-TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/A/0a82e43cb2cb41838b61e97b23444822 to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/A/0a82e43cb2cb41838b61e97b23444822 2024-11-22T03:40:16,000 DEBUG [StoreCloser-TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/A/36f4ff699dd84da1913edc79545b1f8f to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/A/36f4ff699dd84da1913edc79545b1f8f 2024-11-22T03:40:16,001 DEBUG [StoreCloser-TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/A/f7a8dc7598c0453695f04d4f4bdb1c5d to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/A/f7a8dc7598c0453695f04d4f4bdb1c5d 2024-11-22T03:40:16,003 DEBUG [StoreCloser-TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/A/05035c1f745345c3b99912723bf86b20 to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/A/05035c1f745345c3b99912723bf86b20 2024-11-22T03:40:16,004 DEBUG [StoreCloser-TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/A/00f669db6b0b4f248e23aea844f17710 to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/A/00f669db6b0b4f248e23aea844f17710 2024-11-22T03:40:16,005 DEBUG [StoreCloser-TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/A/79873c10df23470e91a8370dd7307ef0 to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/A/79873c10df23470e91a8370dd7307ef0 2024-11-22T03:40:16,007 DEBUG [StoreCloser-TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/A/ab5718b4e15340b08839c04e41e389eb to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/A/ab5718b4e15340b08839c04e41e389eb 2024-11-22T03:40:16,010 DEBUG [StoreCloser-TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/A/c2cfd0728eb0486ebd70c72ce57458ac to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/A/c2cfd0728eb0486ebd70c72ce57458ac 2024-11-22T03:40:16,011 DEBUG [StoreCloser-TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/A/d89aa080a04b44c6b46fa211652eee8f to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/A/d89aa080a04b44c6b46fa211652eee8f 2024-11-22T03:40:16,013 DEBUG [StoreCloser-TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/A/a0cec3316e6c4395ab8f80f15ed72fee to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/A/a0cec3316e6c4395ab8f80f15ed72fee 2024-11-22T03:40:16,014 DEBUG [StoreCloser-TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/A/32714810e79943e489b8965490a9deb0 to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/A/32714810e79943e489b8965490a9deb0 2024-11-22T03:40:16,016 DEBUG [StoreCloser-TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/A/2a450a145c3044f69195ccd86386ed94 to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/A/2a450a145c3044f69195ccd86386ed94 2024-11-22T03:40:16,018 DEBUG [StoreCloser-TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/A/daebc7c44ab644e79e0306a701ccae2c to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/A/daebc7c44ab644e79e0306a701ccae2c 2024-11-22T03:40:16,019 DEBUG [StoreCloser-TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/A/d405092cdcbd408caf5f8951e21805c3 to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/A/d405092cdcbd408caf5f8951e21805c3 2024-11-22T03:40:16,021 DEBUG [StoreCloser-TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/A/d6f08aef05934343b1f738a5442203b6 to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/A/d6f08aef05934343b1f738a5442203b6 2024-11-22T03:40:16,025 DEBUG [StoreCloser-TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/A/c0bd9fec57124d94b2204832a0dea86b to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/A/c0bd9fec57124d94b2204832a0dea86b 2024-11-22T03:40:16,027 DEBUG [StoreCloser-TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/A/5818145b02e14d9f9c0b2427280fb481 to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/A/5818145b02e14d9f9c0b2427280fb481 2024-11-22T03:40:16,029 DEBUG [StoreCloser-TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/A/8102c612cad44458827ed502b2e65903 to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/A/8102c612cad44458827ed502b2e65903 2024-11-22T03:40:16,030 DEBUG [StoreCloser-TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/A/19d5c76c52cb46789f6bf5885b7650e4 to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/A/19d5c76c52cb46789f6bf5885b7650e4 2024-11-22T03:40:16,032 DEBUG [StoreCloser-TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/A/346aa3a7f4c340d2bd5f2f50473fec1d to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/A/346aa3a7f4c340d2bd5f2f50473fec1d 2024-11-22T03:40:16,035 DEBUG [StoreCloser-TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/A/7409d20987cb4ff8b7bf232924241410 to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/A/7409d20987cb4ff8b7bf232924241410 2024-11-22T03:40:16,037 DEBUG [StoreCloser-TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/A/96968a6efacc4168b786ad9f0419e2d2 to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/A/96968a6efacc4168b786ad9f0419e2d2 2024-11-22T03:40:16,039 DEBUG [StoreCloser-TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/A/1012df2ff75844c08fdacb8cdef652f7 to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/A/1012df2ff75844c08fdacb8cdef652f7 2024-11-22T03:40:16,041 DEBUG [StoreCloser-TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/A/4ec35656f37a4029acb763645fd5d7b6 to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/A/4ec35656f37a4029acb763645fd5d7b6 2024-11-22T03:40:16,042 DEBUG [StoreCloser-TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/A/c2e5994cc723423d8c9c71faee245392 to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/A/c2e5994cc723423d8c9c71faee245392 2024-11-22T03:40:16,047 DEBUG [StoreCloser-TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/A/60f4a58a80e149838132f3e12eccb350 to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/A/60f4a58a80e149838132f3e12eccb350 2024-11-22T03:40:16,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-22T03:40:16,065 DEBUG [StoreCloser-TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/B/920b0be291b247aa8d976946b061290c, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/B/7acdd5c62ba64939a5cf0a87243c40ce, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/B/0f128c9d120046de970579dd7ee0bebd, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/B/f4065c9b2c26424fa79e3683eee8f910, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/B/e44e1b6aa5ff4495b4856a90fd6c6d42, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/B/1ff2b194d9194e95884acc70c25b145b, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/B/8431005a11e742fdad7db991e4bb7edf, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/B/5c73ef8267f0487780508fca54b319af, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/B/a7e25a69ac3140e0a87e1b8c0a58c763, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/B/9aa2b44f5aca4b8c990f15b9f83fe55a, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/B/d9f9f6631f4f4fb3862749634fca533f, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/B/cc3ca2a6e42c49eb85d94b4b08fe9104, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/B/a694ac35e4c94035986bf5f8175a2f16, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/B/c976b62d8c01408cb8e1e55b628c7e95, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/B/b5b38c5af776443aa690171ccc4f2d89, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/B/cb1e184b4be04ff88e908f63efa0280d, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/B/78bf0c831baf4229a20ac284a6e5da48, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/B/7aecc4f8d3c345f89aeaaf86b74860d1, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/B/88db1860aa1b443c92cfdfd6a9e386a8, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/B/7cfe5175ae7649eda140430a80603695, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/B/521ceb8dc886461da9ae9e46b5b62b8f, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/B/7e919651fda74c34b20c4a3f74ef0026, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/B/c560f3405cd74dbeb62d9e4b37c3adde, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/B/549bca3e86484bba81f9a9d7402f3d35, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/B/e2f8b73af76f4802822b4a935de88c17, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/B/0950866217f9482c88e3e608290ecd66, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/B/72bf86328be448638eaa8717eaa54a26, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/B/eb8039590ba24f66984c65a34d966d6d, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/B/3a5fb2435eae4c51ba4d913077c59c0e, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/B/d4890a262733475693d2f4112a05d516, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/B/85d6e91de066416c8b4b382e77605b84, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/B/6a3f70e5f9a34c8ba27ce66caac8c258, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/B/6609b977b99c437babb9e14b5b5c0aa2, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/B/47787cb3a6f64dfbae38f5ad071292b1, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/B/101e7d6ab06742c2b5c6fb7586a410ab] to archive 2024-11-22T03:40:16,067 DEBUG [StoreCloser-TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-22T03:40:16,069 DEBUG [StoreCloser-TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/B/920b0be291b247aa8d976946b061290c to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/B/920b0be291b247aa8d976946b061290c 2024-11-22T03:40:16,071 DEBUG [StoreCloser-TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/B/7acdd5c62ba64939a5cf0a87243c40ce to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/B/7acdd5c62ba64939a5cf0a87243c40ce 2024-11-22T03:40:16,073 DEBUG [StoreCloser-TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/B/0f128c9d120046de970579dd7ee0bebd to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/B/0f128c9d120046de970579dd7ee0bebd 2024-11-22T03:40:16,082 DEBUG [StoreCloser-TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/B/f4065c9b2c26424fa79e3683eee8f910 to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/B/f4065c9b2c26424fa79e3683eee8f910 2024-11-22T03:40:16,085 DEBUG [StoreCloser-TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/B/e44e1b6aa5ff4495b4856a90fd6c6d42 to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/B/e44e1b6aa5ff4495b4856a90fd6c6d42 2024-11-22T03:40:16,087 DEBUG [StoreCloser-TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/B/1ff2b194d9194e95884acc70c25b145b to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/B/1ff2b194d9194e95884acc70c25b145b 2024-11-22T03:40:16,089 DEBUG [StoreCloser-TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/B/8431005a11e742fdad7db991e4bb7edf to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/B/8431005a11e742fdad7db991e4bb7edf 2024-11-22T03:40:16,091 DEBUG [StoreCloser-TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/B/5c73ef8267f0487780508fca54b319af to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/B/5c73ef8267f0487780508fca54b319af 2024-11-22T03:40:16,094 DEBUG [StoreCloser-TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/B/a7e25a69ac3140e0a87e1b8c0a58c763 to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/B/a7e25a69ac3140e0a87e1b8c0a58c763 2024-11-22T03:40:16,095 DEBUG [StoreCloser-TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/B/9aa2b44f5aca4b8c990f15b9f83fe55a to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/B/9aa2b44f5aca4b8c990f15b9f83fe55a 2024-11-22T03:40:16,098 DEBUG [StoreCloser-TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/B/d9f9f6631f4f4fb3862749634fca533f to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/B/d9f9f6631f4f4fb3862749634fca533f 2024-11-22T03:40:16,100 DEBUG [StoreCloser-TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/B/cc3ca2a6e42c49eb85d94b4b08fe9104 to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/B/cc3ca2a6e42c49eb85d94b4b08fe9104 2024-11-22T03:40:16,102 DEBUG [StoreCloser-TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/B/a694ac35e4c94035986bf5f8175a2f16 to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/B/a694ac35e4c94035986bf5f8175a2f16 2024-11-22T03:40:16,103 DEBUG [StoreCloser-TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/B/c976b62d8c01408cb8e1e55b628c7e95 to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/B/c976b62d8c01408cb8e1e55b628c7e95 2024-11-22T03:40:16,105 DEBUG [StoreCloser-TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/B/b5b38c5af776443aa690171ccc4f2d89 to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/B/b5b38c5af776443aa690171ccc4f2d89 2024-11-22T03:40:16,106 DEBUG [StoreCloser-TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/B/cb1e184b4be04ff88e908f63efa0280d to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/B/cb1e184b4be04ff88e908f63efa0280d 2024-11-22T03:40:16,108 DEBUG [StoreCloser-TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/B/78bf0c831baf4229a20ac284a6e5da48 to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/B/78bf0c831baf4229a20ac284a6e5da48 2024-11-22T03:40:16,110 DEBUG [StoreCloser-TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/B/7aecc4f8d3c345f89aeaaf86b74860d1 to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/B/7aecc4f8d3c345f89aeaaf86b74860d1 2024-11-22T03:40:16,112 DEBUG [StoreCloser-TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/B/88db1860aa1b443c92cfdfd6a9e386a8 to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/B/88db1860aa1b443c92cfdfd6a9e386a8 2024-11-22T03:40:16,113 DEBUG [StoreCloser-TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/B/7cfe5175ae7649eda140430a80603695 to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/B/7cfe5175ae7649eda140430a80603695 2024-11-22T03:40:16,116 DEBUG [StoreCloser-TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/B/521ceb8dc886461da9ae9e46b5b62b8f to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/B/521ceb8dc886461da9ae9e46b5b62b8f 2024-11-22T03:40:16,118 DEBUG [StoreCloser-TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/B/7e919651fda74c34b20c4a3f74ef0026 to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/B/7e919651fda74c34b20c4a3f74ef0026 2024-11-22T03:40:16,119 DEBUG [StoreCloser-TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/B/c560f3405cd74dbeb62d9e4b37c3adde to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/B/c560f3405cd74dbeb62d9e4b37c3adde 2024-11-22T03:40:16,120 DEBUG [StoreCloser-TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/B/549bca3e86484bba81f9a9d7402f3d35 to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/B/549bca3e86484bba81f9a9d7402f3d35 2024-11-22T03:40:16,122 DEBUG [StoreCloser-TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/B/e2f8b73af76f4802822b4a935de88c17 to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/B/e2f8b73af76f4802822b4a935de88c17 2024-11-22T03:40:16,123 DEBUG [StoreCloser-TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/B/0950866217f9482c88e3e608290ecd66 to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/B/0950866217f9482c88e3e608290ecd66 2024-11-22T03:40:16,125 DEBUG [StoreCloser-TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/B/72bf86328be448638eaa8717eaa54a26 to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/B/72bf86328be448638eaa8717eaa54a26 2024-11-22T03:40:16,126 DEBUG [StoreCloser-TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/B/eb8039590ba24f66984c65a34d966d6d to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/B/eb8039590ba24f66984c65a34d966d6d 2024-11-22T03:40:16,128 DEBUG [StoreCloser-TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/B/3a5fb2435eae4c51ba4d913077c59c0e to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/B/3a5fb2435eae4c51ba4d913077c59c0e 2024-11-22T03:40:16,130 DEBUG [StoreCloser-TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/B/d4890a262733475693d2f4112a05d516 to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/B/d4890a262733475693d2f4112a05d516 2024-11-22T03:40:16,131 DEBUG [StoreCloser-TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/B/85d6e91de066416c8b4b382e77605b84 to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/B/85d6e91de066416c8b4b382e77605b84 2024-11-22T03:40:16,133 DEBUG [StoreCloser-TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/B/6a3f70e5f9a34c8ba27ce66caac8c258 to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/B/6a3f70e5f9a34c8ba27ce66caac8c258 2024-11-22T03:40:16,134 DEBUG [StoreCloser-TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/B/6609b977b99c437babb9e14b5b5c0aa2 to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/B/6609b977b99c437babb9e14b5b5c0aa2 2024-11-22T03:40:16,135 DEBUG [StoreCloser-TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/B/47787cb3a6f64dfbae38f5ad071292b1 to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/B/47787cb3a6f64dfbae38f5ad071292b1 2024-11-22T03:40:16,138 DEBUG [StoreCloser-TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/B/101e7d6ab06742c2b5c6fb7586a410ab to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/B/101e7d6ab06742c2b5c6fb7586a410ab 2024-11-22T03:40:16,140 DEBUG [StoreCloser-TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/C/8a9d616772a64f4294051324e940a5ca, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/C/eda6acb3909c4de0a95ab22dd66ca079, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/C/fca97e2fbc9b4dc3a0a8476e5c0efc36, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/C/e232665be81a42d3b8e5ab310626de53, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/C/7aac20e45d8d40d6a4db4a305f6d6470, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/C/50d5d6ce1cb344e799319490c37f9467, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/C/4bf36c6e00ea49f5a9436984eeb27813, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/C/bcd10c89b3cd4c72a3058f5feace7274, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/C/fe675d3c14a347b89b6378844726f367, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/C/94b34a12a19745dc85d07bbbabd0d6b3, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/C/515908705c6f445bb08d4e794c7576ac, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/C/d6dbee770d1146b8bf3a9dc44312317f, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/C/60e84e1c86ad4cd3aaa3d684cc06a3ad, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/C/a34d358420cf44e69c2304c411cb8b83, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/C/6fae8a2e9abd4328acd0050de197d16c, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/C/790cd2d9acd542e09d2c8d1c49631c80, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/C/5ff0a1cd4e424f10bffb4253faeb07d0, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/C/4a506faf14c64ba0be946fea194981f9, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/C/9502f711563044d5bcdc7557806f0d89, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/C/477966aea94140e98f95e2921a241b01, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/C/5a207447ec7d4403b1520d0267c51f62, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/C/3b798870b3c44845916d55d82f80abb4, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/C/081b196b6c2c4cacaf395699683f5b84, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/C/e15a26d1669c4c1baa758db1a82ca2a3, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/C/b2dd03f386d44f348130c9816093e5b6, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/C/60fdfd0728e94e19b2cbd98d852f28b8, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/C/3f329e65bc024cb0916a0b3b969f4fb2, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/C/dd8cd07fc06040c98cc9a47155304fa0, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/C/7f2dbb2fab82448dabdfdcad1b3d7749, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/C/c6e1e26de0964d538ca712887fcc5a2f, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/C/fe573883a1474be38ffd221a6ef3d57d, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/C/8656d12e45e248bc838b318639c7a673, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/C/459847ab448e480f929edc922aa3927c, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/C/8379710d47b946d2bafadda98cb2ed87, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/C/b11298ed22f943ed8cd400e2ac8fc876] to archive 2024-11-22T03:40:16,141 DEBUG [StoreCloser-TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-22T03:40:16,147 DEBUG [StoreCloser-TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/C/8a9d616772a64f4294051324e940a5ca to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/C/8a9d616772a64f4294051324e940a5ca 2024-11-22T03:40:16,148 DEBUG [StoreCloser-TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/C/eda6acb3909c4de0a95ab22dd66ca079 to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/C/eda6acb3909c4de0a95ab22dd66ca079 2024-11-22T03:40:16,150 DEBUG [StoreCloser-TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/C/fca97e2fbc9b4dc3a0a8476e5c0efc36 to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/C/fca97e2fbc9b4dc3a0a8476e5c0efc36 2024-11-22T03:40:16,151 DEBUG [StoreCloser-TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/C/e232665be81a42d3b8e5ab310626de53 to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/C/e232665be81a42d3b8e5ab310626de53 2024-11-22T03:40:16,152 DEBUG [StoreCloser-TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/C/7aac20e45d8d40d6a4db4a305f6d6470 to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/C/7aac20e45d8d40d6a4db4a305f6d6470 2024-11-22T03:40:16,154 DEBUG [StoreCloser-TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/C/50d5d6ce1cb344e799319490c37f9467 to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/C/50d5d6ce1cb344e799319490c37f9467 2024-11-22T03:40:16,155 DEBUG [StoreCloser-TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/C/4bf36c6e00ea49f5a9436984eeb27813 to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/C/4bf36c6e00ea49f5a9436984eeb27813 2024-11-22T03:40:16,156 DEBUG [StoreCloser-TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/C/bcd10c89b3cd4c72a3058f5feace7274 to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/C/bcd10c89b3cd4c72a3058f5feace7274 2024-11-22T03:40:16,158 DEBUG [StoreCloser-TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/C/fe675d3c14a347b89b6378844726f367 to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/C/fe675d3c14a347b89b6378844726f367 2024-11-22T03:40:16,159 DEBUG [StoreCloser-TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/C/94b34a12a19745dc85d07bbbabd0d6b3 to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/C/94b34a12a19745dc85d07bbbabd0d6b3 2024-11-22T03:40:16,161 DEBUG [StoreCloser-TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/C/515908705c6f445bb08d4e794c7576ac to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/C/515908705c6f445bb08d4e794c7576ac 2024-11-22T03:40:16,163 DEBUG [StoreCloser-TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/C/d6dbee770d1146b8bf3a9dc44312317f to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/C/d6dbee770d1146b8bf3a9dc44312317f 2024-11-22T03:40:16,165 DEBUG [StoreCloser-TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/C/60e84e1c86ad4cd3aaa3d684cc06a3ad to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/C/60e84e1c86ad4cd3aaa3d684cc06a3ad 2024-11-22T03:40:16,166 DEBUG [StoreCloser-TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/C/a34d358420cf44e69c2304c411cb8b83 to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/C/a34d358420cf44e69c2304c411cb8b83 2024-11-22T03:40:16,168 DEBUG [StoreCloser-TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/C/6fae8a2e9abd4328acd0050de197d16c to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/C/6fae8a2e9abd4328acd0050de197d16c 2024-11-22T03:40:16,170 DEBUG [StoreCloser-TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/C/790cd2d9acd542e09d2c8d1c49631c80 to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/C/790cd2d9acd542e09d2c8d1c49631c80 2024-11-22T03:40:16,171 DEBUG [StoreCloser-TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/C/5ff0a1cd4e424f10bffb4253faeb07d0 to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/C/5ff0a1cd4e424f10bffb4253faeb07d0 2024-11-22T03:40:16,173 DEBUG [StoreCloser-TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/C/4a506faf14c64ba0be946fea194981f9 to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/C/4a506faf14c64ba0be946fea194981f9 2024-11-22T03:40:16,175 DEBUG [StoreCloser-TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/C/9502f711563044d5bcdc7557806f0d89 to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/C/9502f711563044d5bcdc7557806f0d89 2024-11-22T03:40:16,177 DEBUG [StoreCloser-TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/C/477966aea94140e98f95e2921a241b01 to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/C/477966aea94140e98f95e2921a241b01 2024-11-22T03:40:16,178 DEBUG [StoreCloser-TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/C/5a207447ec7d4403b1520d0267c51f62 to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/C/5a207447ec7d4403b1520d0267c51f62 2024-11-22T03:40:16,180 DEBUG [StoreCloser-TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/C/3b798870b3c44845916d55d82f80abb4 to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/C/3b798870b3c44845916d55d82f80abb4 2024-11-22T03:40:16,183 DEBUG [StoreCloser-TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/C/081b196b6c2c4cacaf395699683f5b84 to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/C/081b196b6c2c4cacaf395699683f5b84 2024-11-22T03:40:16,186 DEBUG [StoreCloser-TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/C/e15a26d1669c4c1baa758db1a82ca2a3 to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/C/e15a26d1669c4c1baa758db1a82ca2a3 2024-11-22T03:40:16,188 DEBUG [StoreCloser-TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/C/b2dd03f386d44f348130c9816093e5b6 to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/C/b2dd03f386d44f348130c9816093e5b6 2024-11-22T03:40:16,194 DEBUG [StoreCloser-TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/C/60fdfd0728e94e19b2cbd98d852f28b8 to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/C/60fdfd0728e94e19b2cbd98d852f28b8 2024-11-22T03:40:16,197 DEBUG [StoreCloser-TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/C/3f329e65bc024cb0916a0b3b969f4fb2 to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/C/3f329e65bc024cb0916a0b3b969f4fb2 2024-11-22T03:40:16,199 DEBUG [StoreCloser-TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/C/dd8cd07fc06040c98cc9a47155304fa0 to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/C/dd8cd07fc06040c98cc9a47155304fa0 2024-11-22T03:40:16,201 DEBUG [StoreCloser-TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/C/7f2dbb2fab82448dabdfdcad1b3d7749 to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/C/7f2dbb2fab82448dabdfdcad1b3d7749 2024-11-22T03:40:16,203 DEBUG [StoreCloser-TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/C/c6e1e26de0964d538ca712887fcc5a2f to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/C/c6e1e26de0964d538ca712887fcc5a2f 2024-11-22T03:40:16,204 DEBUG [StoreCloser-TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/C/fe573883a1474be38ffd221a6ef3d57d to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/C/fe573883a1474be38ffd221a6ef3d57d 2024-11-22T03:40:16,206 DEBUG [StoreCloser-TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/C/8656d12e45e248bc838b318639c7a673 to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/C/8656d12e45e248bc838b318639c7a673 2024-11-22T03:40:16,207 DEBUG [StoreCloser-TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/C/459847ab448e480f929edc922aa3927c to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/C/459847ab448e480f929edc922aa3927c 2024-11-22T03:40:16,208 DEBUG [StoreCloser-TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/C/8379710d47b946d2bafadda98cb2ed87 to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/C/8379710d47b946d2bafadda98cb2ed87 2024-11-22T03:40:16,209 DEBUG [StoreCloser-TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/C/b11298ed22f943ed8cd400e2ac8fc876 to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/C/b11298ed22f943ed8cd400e2ac8fc876 2024-11-22T03:40:16,214 DEBUG [RS_CLOSE_REGION-regionserver/c85114ed5096:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/recovered.edits/539.seqid, newMaxSeqId=539, maxSeqId=1 2024-11-22T03:40:16,217 INFO [RS_CLOSE_REGION-regionserver/c85114ed5096:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4. 2024-11-22T03:40:16,217 DEBUG [RS_CLOSE_REGION-regionserver/c85114ed5096:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1676): Region close journal for f970bd9f1e861e2b64a2fb87fb209ac4: Waiting for close lock at 1732246814069Running coprocessor pre-close hooks at 1732246814069Disabling compacts and flushes for region at 1732246814069Disabling writes for close at 1732246814685 (+616 ms)Obtaining lock to block concurrent updates at 1732246814685Preparing flush snapshotting stores in f970bd9f1e861e2b64a2fb87fb209ac4 at 1732246814685Finished memstore snapshotting TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4., syncing WAL and waiting on mvcc, flushsize=dataSize=27480, getHeapSize=72720, getOffHeapSize=0, getCellsCount=600 at 1732246814686 (+1 ms)Flushing stores of TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4. at 1732246814686Flushing f970bd9f1e861e2b64a2fb87fb209ac4/A: creating writer at 1732246814686Flushing f970bd9f1e861e2b64a2fb87fb209ac4/A: appending metadata at 1732246814689 (+3 ms)Flushing f970bd9f1e861e2b64a2fb87fb209ac4/A: closing flushed file at 1732246814689Flushing f970bd9f1e861e2b64a2fb87fb209ac4/B: creating writer at 1732246815099 (+410 ms)Flushing f970bd9f1e861e2b64a2fb87fb209ac4/B: appending metadata at 1732246815102 (+3 ms)Flushing f970bd9f1e861e2b64a2fb87fb209ac4/B: closing flushed file at 1732246815102Flushing f970bd9f1e861e2b64a2fb87fb209ac4/C: creating writer at 1732246815514 (+412 ms)Flushing f970bd9f1e861e2b64a2fb87fb209ac4/C: appending metadata at 1732246815516 (+2 ms)Flushing f970bd9f1e861e2b64a2fb87fb209ac4/C: closing flushed file at 1732246815517 (+1 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6e99c326: reopening flushed file at 1732246815927 (+410 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@c9d083c: reopening flushed file at 1732246815939 (+12 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4a888cad: reopening flushed file at 1732246815949 (+10 ms)Finished flush of dataSize ~26.84 KB/27480, heapSize ~71.02 KB/72720, currentSize=0 B/0 for f970bd9f1e861e2b64a2fb87fb209ac4 in 1271ms, sequenceid=536, compaction requested=true at 1732246815956 (+7 ms)Writing region close event to WAL at 1732246816211 (+255 ms)Running coprocessor post-close hooks at 1732246816215 (+4 ms)Closed at 1732246816217 (+2 ms) 2024-11-22T03:40:16,219 INFO [RS_CLOSE_REGION-regionserver/c85114ed5096:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] handler.UnassignRegionHandler(157): Closed f970bd9f1e861e2b64a2fb87fb209ac4 2024-11-22T03:40:16,220 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=31 updating hbase:meta row=f970bd9f1e861e2b64a2fb87fb209ac4, regionState=CLOSED 2024-11-22T03:40:16,222 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=32, ppid=31, state=RUNNABLE, hasLock=false; CloseRegionProcedure f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 because future has completed 2024-11-22T03:40:16,224 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=32, resume processing ppid=31 2024-11-22T03:40:16,224 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=32, ppid=31, state=SUCCESS, hasLock=false; CloseRegionProcedure f970bd9f1e861e2b64a2fb87fb209ac4, server=c85114ed5096,42545,1732246786482 in 2.3110 sec 2024-11-22T03:40:16,226 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=31, resume processing ppid=30 2024-11-22T03:40:16,226 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=31, ppid=30, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f970bd9f1e861e2b64a2fb87fb209ac4, UNASSIGN in 2.3180 sec 2024-11-22T03:40:16,229 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=30, resume processing ppid=29 2024-11-22T03:40:16,229 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=30, ppid=29, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 2.3260 sec 2024-11-22T03:40:16,230 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732246816230"}]},"ts":"1732246816230"} 2024-11-22T03:40:16,232 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-22T03:40:16,232 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-22T03:40:16,234 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=29, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 2.3430 sec 2024-11-22T03:40:18,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-22T03:40:18,059 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-22T03:40:18,063 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42627 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-22T03:40:18,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42627 {}] procedure2.ProcedureExecutor(1139): Stored pid=33, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-22T03:40:18,069 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=33, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-22T03:40:18,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=33 2024-11-22T03:40:18,070 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=33, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-22T03:40:18,076 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4 2024-11-22T03:40:18,081 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/A, FileablePath, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/B, FileablePath, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/C, FileablePath, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/recovered.edits] 2024-11-22T03:40:18,085 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/A/497f5b96b7d14648a6380d0146a6b40d to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/A/497f5b96b7d14648a6380d0146a6b40d 2024-11-22T03:40:18,087 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/A/5ef48e3dd17d48568d9244f882b240a6 to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/A/5ef48e3dd17d48568d9244f882b240a6 2024-11-22T03:40:18,089 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/A/c24c7f9ffe744e84b219368ab3e57429 to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/A/c24c7f9ffe744e84b219368ab3e57429 2024-11-22T03:40:18,090 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/A/fbe68947faba4993a94d066c07104d6d to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/A/fbe68947faba4993a94d066c07104d6d 2024-11-22T03:40:18,092 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/B/15aef831f06b41048e24feb572503cf6 to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/B/15aef831f06b41048e24feb572503cf6 2024-11-22T03:40:18,094 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/B/20cda7031cd546f9a7d2f73ae220bada to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/B/20cda7031cd546f9a7d2f73ae220bada 2024-11-22T03:40:18,095 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/B/774c2a8dd66146aa960f86b2ee71ad2c to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/B/774c2a8dd66146aa960f86b2ee71ad2c 2024-11-22T03:40:18,096 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/B/9f70bddec72c4126b2cf014172f254c4 to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/B/9f70bddec72c4126b2cf014172f254c4 2024-11-22T03:40:18,099 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/C/2608aa120f7b4c91a5318ac30f6f7fce to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/C/2608aa120f7b4c91a5318ac30f6f7fce 2024-11-22T03:40:18,100 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/C/57217b1e38824a8fb207307d5ed9baa1 to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/C/57217b1e38824a8fb207307d5ed9baa1 2024-11-22T03:40:18,101 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/C/b71215a78e0844cbb6df1a176d4fcfcc to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/C/b71215a78e0844cbb6df1a176d4fcfcc 2024-11-22T03:40:18,102 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/C/cc39764ab6e54cf39cfc8fd0bd0dc4f2 to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/C/cc39764ab6e54cf39cfc8fd0bd0dc4f2 2024-11-22T03:40:18,105 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/recovered.edits/539.seqid to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4/recovered.edits/539.seqid 2024-11-22T03:40:18,106 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f970bd9f1e861e2b64a2fb87fb209ac4 2024-11-22T03:40:18,106 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-22T03:40:18,111 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=33, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-22T03:40:18,115 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42545 {}] util.ReflectedFunctionCache(97): Populated cache for org.apache.hadoop.hbase.filter.KeyOnlyFilter in 0ms 2024-11-22T03:40:18,122 WARN [PEWorker-3 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-22T03:40:18,157 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-22T03:40:18,158 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=33, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-22T03:40:18,158 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-22T03:40:18,159 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732246818158"}]},"ts":"9223372036854775807"} 2024-11-22T03:40:18,163 INFO [PEWorker-3 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-22T03:40:18,163 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => f970bd9f1e861e2b64a2fb87fb209ac4, NAME => 'TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4.', STARTKEY => '', ENDKEY => ''}] 2024-11-22T03:40:18,163 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-22T03:40:18,164 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732246818163"}]},"ts":"9223372036854775807"} 2024-11-22T03:40:18,167 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-22T03:40:18,168 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(133): Finished pid=33, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-22T03:40:18,169 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=33, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 105 msec 2024-11-22T03:40:18,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=33 2024-11-22T03:40:18,179 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-22T03:40:18,179 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-22T03:40:18,190 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testMixedAtomicity Thread=239 (was 211) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-10 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-9 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1251162740_22 at /127.0.0.1:53422 [Waiting for operation #276] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-13 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-14 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-5 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-12 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-16 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-8 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS:0;c85114ed5096:42545-shortCompactions-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.PriorityBlockingQueue.take(PriorityBlockingQueue.java:535) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-15 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-11 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_CLOSE_REGION-regionserver/c85114ed5096:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_OPEN_REGION-regionserver/c85114ed5096:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1251162740_22 at /127.0.0.1:46806 [Waiting for operation #292] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1145069759_22 at /127.0.0.1:53364 [Waiting for operation #166] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=459 (was 441) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=446 (was 363) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=3563 (was 4033) 2024-11-22T03:40:18,198 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testMobMixedAtomicity Thread=239, OpenFileDescriptor=459, MaxFileDescriptor=1048576, SystemLoadAverage=446, ProcessCount=11, AvailableMemoryMB=3560 2024-11-22T03:40:18,199 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42627 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-22T03:40:18,199 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42627 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-22T03:40:18,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42627 {}] procedure2.ProcedureExecutor(1139): Stored pid=34, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-22T03:40:18,202 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=34, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-22T03:40:18,202 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:18,202 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42627 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 34 2024-11-22T03:40:18,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-11-22T03:40:18,203 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=34, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-22T03:40:18,209 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073741954_1130 (size=959) 2024-11-22T03:40:18,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-11-22T03:40:18,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-11-22T03:40:18,614 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 53a70bd4899ab7aa137c5ffbd98f8de7, NAME => 'TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a 2024-11-22T03:40:18,622 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073741955_1131 (size=53) 2024-11-22T03:40:18,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-11-22T03:40:19,024 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-22T03:40:19,024 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 53a70bd4899ab7aa137c5ffbd98f8de7, disabling compactions & flushes 2024-11-22T03:40:19,024 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7. 2024-11-22T03:40:19,025 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7. 2024-11-22T03:40:19,025 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7. after waiting 0 ms 2024-11-22T03:40:19,025 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7. 2024-11-22T03:40:19,025 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7. 2024-11-22T03:40:19,025 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 53a70bd4899ab7aa137c5ffbd98f8de7: Waiting for close lock at 1732246819024Disabling compacts and flushes for region at 1732246819024Disabling writes for close at 1732246819025 (+1 ms)Writing region close event to WAL at 1732246819025Closed at 1732246819025 2024-11-22T03:40:19,028 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=34, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-22T03:40:19,029 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732246819028"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732246819028"}]},"ts":"1732246819028"} 2024-11-22T03:40:19,032 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-22T03:40:19,034 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=34, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-22T03:40:19,034 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732246819034"}]},"ts":"1732246819034"} 2024-11-22T03:40:19,037 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-22T03:40:19,037 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=53a70bd4899ab7aa137c5ffbd98f8de7, ASSIGN}] 2024-11-22T03:40:19,038 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=53a70bd4899ab7aa137c5ffbd98f8de7, ASSIGN 2024-11-22T03:40:19,039 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=53a70bd4899ab7aa137c5ffbd98f8de7, ASSIGN; state=OFFLINE, location=c85114ed5096,42545,1732246786482; forceNewPlan=false, retain=false 2024-11-22T03:40:19,190 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=35 updating hbase:meta row=53a70bd4899ab7aa137c5ffbd98f8de7, regionState=OPENING, regionLocation=c85114ed5096,42545,1732246786482 2024-11-22T03:40:19,192 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=53a70bd4899ab7aa137c5ffbd98f8de7, ASSIGN because future has completed 2024-11-22T03:40:19,193 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=36, ppid=35, state=RUNNABLE, hasLock=false; OpenRegionProcedure 53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482}] 2024-11-22T03:40:19,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-11-22T03:40:19,352 INFO [RS_OPEN_REGION-regionserver/c85114ed5096:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7. 2024-11-22T03:40:19,353 DEBUG [RS_OPEN_REGION-regionserver/c85114ed5096:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(7752): Opening region: {ENCODED => 53a70bd4899ab7aa137c5ffbd98f8de7, NAME => 'TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7.', STARTKEY => '', ENDKEY => ''} 2024-11-22T03:40:19,353 DEBUG [RS_OPEN_REGION-regionserver/c85114ed5096:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 53a70bd4899ab7aa137c5ffbd98f8de7 2024-11-22T03:40:19,354 DEBUG [RS_OPEN_REGION-regionserver/c85114ed5096:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-22T03:40:19,354 DEBUG [RS_OPEN_REGION-regionserver/c85114ed5096:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(7794): checking encryption for 53a70bd4899ab7aa137c5ffbd98f8de7 2024-11-22T03:40:19,354 DEBUG [RS_OPEN_REGION-regionserver/c85114ed5096:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(7797): checking classloading for 53a70bd4899ab7aa137c5ffbd98f8de7 2024-11-22T03:40:19,357 INFO [StoreOpener-53a70bd4899ab7aa137c5ffbd98f8de7-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 53a70bd4899ab7aa137c5ffbd98f8de7 2024-11-22T03:40:19,360 INFO [StoreOpener-53a70bd4899ab7aa137c5ffbd98f8de7-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 53a70bd4899ab7aa137c5ffbd98f8de7 columnFamilyName A 2024-11-22T03:40:19,360 DEBUG [StoreOpener-53a70bd4899ab7aa137c5ffbd98f8de7-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:19,361 INFO [StoreOpener-53a70bd4899ab7aa137c5ffbd98f8de7-1 {}] regionserver.HStore(327): Store=53a70bd4899ab7aa137c5ffbd98f8de7/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-22T03:40:19,361 INFO [StoreOpener-53a70bd4899ab7aa137c5ffbd98f8de7-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 53a70bd4899ab7aa137c5ffbd98f8de7 2024-11-22T03:40:19,363 INFO [StoreOpener-53a70bd4899ab7aa137c5ffbd98f8de7-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 53a70bd4899ab7aa137c5ffbd98f8de7 columnFamilyName B 2024-11-22T03:40:19,363 DEBUG [StoreOpener-53a70bd4899ab7aa137c5ffbd98f8de7-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:19,363 INFO [StoreOpener-53a70bd4899ab7aa137c5ffbd98f8de7-1 {}] regionserver.HStore(327): Store=53a70bd4899ab7aa137c5ffbd98f8de7/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-22T03:40:19,363 INFO [StoreOpener-53a70bd4899ab7aa137c5ffbd98f8de7-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 53a70bd4899ab7aa137c5ffbd98f8de7 2024-11-22T03:40:19,365 INFO [StoreOpener-53a70bd4899ab7aa137c5ffbd98f8de7-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 53a70bd4899ab7aa137c5ffbd98f8de7 columnFamilyName C 2024-11-22T03:40:19,365 DEBUG [StoreOpener-53a70bd4899ab7aa137c5ffbd98f8de7-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:19,365 INFO [StoreOpener-53a70bd4899ab7aa137c5ffbd98f8de7-1 {}] regionserver.HStore(327): Store=53a70bd4899ab7aa137c5ffbd98f8de7/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-22T03:40:19,366 DEBUG [RS_OPEN_REGION-regionserver/c85114ed5096:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1038): replaying wal for 53a70bd4899ab7aa137c5ffbd98f8de7 2024-11-22T03:40:19,367 DEBUG [RS_OPEN_REGION-regionserver/c85114ed5096:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7 2024-11-22T03:40:19,367 DEBUG [RS_OPEN_REGION-regionserver/c85114ed5096:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7 2024-11-22T03:40:19,368 DEBUG [RS_OPEN_REGION-regionserver/c85114ed5096:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1048): stopping wal replay for 53a70bd4899ab7aa137c5ffbd98f8de7 2024-11-22T03:40:19,368 DEBUG [RS_OPEN_REGION-regionserver/c85114ed5096:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1060): Cleaning up temporary data for 53a70bd4899ab7aa137c5ffbd98f8de7 2024-11-22T03:40:19,369 DEBUG [RS_OPEN_REGION-regionserver/c85114ed5096:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-22T03:40:19,370 DEBUG [RS_OPEN_REGION-regionserver/c85114ed5096:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1093): writing seq id for 53a70bd4899ab7aa137c5ffbd98f8de7 2024-11-22T03:40:19,373 DEBUG [RS_OPEN_REGION-regionserver/c85114ed5096:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-22T03:40:19,374 INFO [RS_OPEN_REGION-regionserver/c85114ed5096:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1114): Opened 53a70bd4899ab7aa137c5ffbd98f8de7; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62577576, jitterRate=-0.06752145290374756}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-22T03:40:19,374 DEBUG [RS_OPEN_REGION-regionserver/c85114ed5096:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 53a70bd4899ab7aa137c5ffbd98f8de7 2024-11-22T03:40:19,376 DEBUG [RS_OPEN_REGION-regionserver/c85114ed5096:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1006): Region open journal for 53a70bd4899ab7aa137c5ffbd98f8de7: Running coprocessor pre-open hook at 1732246819354Writing region info on filesystem at 1732246819354Initializing all the Stores at 1732246819356 (+2 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732246819356Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732246819356Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732246819356Cleaning up temporary data from old regions at 1732246819368 (+12 ms)Running coprocessor post-open hooks at 1732246819374 (+6 ms)Region opened successfully at 1732246819375 (+1 ms) 2024-11-22T03:40:19,377 INFO [RS_OPEN_REGION-regionserver/c85114ed5096:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., pid=36, masterSystemTime=1732246819345 2024-11-22T03:40:19,379 DEBUG [RS_OPEN_REGION-regionserver/c85114ed5096:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7. 2024-11-22T03:40:19,379 INFO [RS_OPEN_REGION-regionserver/c85114ed5096:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7. 2024-11-22T03:40:19,380 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=35 updating hbase:meta row=53a70bd4899ab7aa137c5ffbd98f8de7, regionState=OPEN, openSeqNum=2, regionLocation=c85114ed5096,42545,1732246786482 2024-11-22T03:40:19,382 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=36, ppid=35, state=RUNNABLE, hasLock=false; OpenRegionProcedure 53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 because future has completed 2024-11-22T03:40:19,386 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=36, resume processing ppid=35 2024-11-22T03:40:19,386 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=36, ppid=35, state=SUCCESS, hasLock=false; OpenRegionProcedure 53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 in 191 msec 2024-11-22T03:40:19,388 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=35, resume processing ppid=34 2024-11-22T03:40:19,389 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=35, ppid=34, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=53a70bd4899ab7aa137c5ffbd98f8de7, ASSIGN in 349 msec 2024-11-22T03:40:19,389 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=34, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-22T03:40:19,390 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732246819389"}]},"ts":"1732246819389"} 2024-11-22T03:40:19,391 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-22T03:40:19,393 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=34, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-22T03:40:19,396 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=34, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1950 sec 2024-11-22T03:40:20,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-11-22T03:40:20,349 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-22T03:40:20,350 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@736030eb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T03:40:20,350 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c85114ed5096,42627,-1 for getting cluster id 2024-11-22T03:40:20,350 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-22T03:40:20,352 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '67035b7f-c649-44a8-9db7-17adff49cedb' 2024-11-22T03:40:20,352 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-22T03:40:20,353 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "67035b7f-c649-44a8-9db7-17adff49cedb" 2024-11-22T03:40:20,353 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2463c28a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T03:40:20,353 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c85114ed5096,42627,-1] 2024-11-22T03:40:20,353 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-22T03:40:20,354 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T03:40:20,355 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53858, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-22T03:40:20,356 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@64f53090, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T03:40:20,356 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-22T03:40:20,357 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c85114ed5096,42545,1732246786482, seqNum=-1] 2024-11-22T03:40:20,358 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-22T03:40:20,359 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34674, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-22T03:40:20,361 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-22T03:40:20,362 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncConnectionImpl(321): The fetched master address is c85114ed5096,42627,1732246785796 2024-11-22T03:40:20,362 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@44bb855c 2024-11-22T03:40:20,362 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-22T03:40:20,363 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53874, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-22T03:40:20,369 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42627 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-22T03:40:20,369 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42627 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-22T03:40:20,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42627 {}] procedure2.ProcedureExecutor(1139): Stored pid=37, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-22T03:40:20,389 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073741956_1132 (size=995) 2024-11-22T03:40:20,791 DEBUG [PEWorker-1 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.959 2024-11-22T03:40:20,791 INFO [PEWorker-1 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.995 2024-11-22T03:40:20,796 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=38, ppid=37, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-22T03:40:20,805 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=39, ppid=38, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=53a70bd4899ab7aa137c5ffbd98f8de7, REOPEN/MOVE}] 2024-11-22T03:40:20,807 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=39, ppid=38, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=53a70bd4899ab7aa137c5ffbd98f8de7, REOPEN/MOVE 2024-11-22T03:40:20,807 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=39 updating hbase:meta row=53a70bd4899ab7aa137c5ffbd98f8de7, regionState=CLOSING, regionLocation=c85114ed5096,42545,1732246786482 2024-11-22T03:40:20,809 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=39, ppid=38, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=53a70bd4899ab7aa137c5ffbd98f8de7, REOPEN/MOVE because future has completed 2024-11-22T03:40:20,810 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-22T03:40:20,810 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=40, ppid=39, state=RUNNABLE, hasLock=false; CloseRegionProcedure 53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482}] 2024-11-22T03:40:20,963 INFO [RS_CLOSE_REGION-regionserver/c85114ed5096:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] handler.UnassignRegionHandler(122): Close 53a70bd4899ab7aa137c5ffbd98f8de7 2024-11-22T03:40:20,963 DEBUG [RS_CLOSE_REGION-regionserver/c85114ed5096:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-22T03:40:20,964 DEBUG [RS_CLOSE_REGION-regionserver/c85114ed5096:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1722): Closing 53a70bd4899ab7aa137c5ffbd98f8de7, disabling compactions & flushes 2024-11-22T03:40:20,964 INFO [RS_CLOSE_REGION-regionserver/c85114ed5096:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7. 2024-11-22T03:40:20,964 DEBUG [RS_CLOSE_REGION-regionserver/c85114ed5096:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7. 2024-11-22T03:40:20,964 DEBUG [RS_CLOSE_REGION-regionserver/c85114ed5096:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7. after waiting 0 ms 2024-11-22T03:40:20,964 DEBUG [RS_CLOSE_REGION-regionserver/c85114ed5096:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7. 2024-11-22T03:40:20,969 DEBUG [RS_CLOSE_REGION-regionserver/c85114ed5096:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-22T03:40:20,970 INFO [RS_CLOSE_REGION-regionserver/c85114ed5096:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7. 2024-11-22T03:40:20,970 DEBUG [RS_CLOSE_REGION-regionserver/c85114ed5096:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1676): Region close journal for 53a70bd4899ab7aa137c5ffbd98f8de7: Waiting for close lock at 1732246820964Running coprocessor pre-close hooks at 1732246820964Disabling compacts and flushes for region at 1732246820964Disabling writes for close at 1732246820964Writing region close event to WAL at 1732246820964Running coprocessor post-close hooks at 1732246820970 (+6 ms)Closed at 1732246820970 2024-11-22T03:40:20,970 WARN [RS_CLOSE_REGION-regionserver/c85114ed5096:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegionServer(3299): Not adding moved region record: 53a70bd4899ab7aa137c5ffbd98f8de7 to self. 2024-11-22T03:40:20,973 INFO [RS_CLOSE_REGION-regionserver/c85114ed5096:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] handler.UnassignRegionHandler(157): Closed 53a70bd4899ab7aa137c5ffbd98f8de7 2024-11-22T03:40:20,974 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=39 updating hbase:meta row=53a70bd4899ab7aa137c5ffbd98f8de7, regionState=CLOSED 2024-11-22T03:40:20,976 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=40, ppid=39, state=RUNNABLE, hasLock=false; CloseRegionProcedure 53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 because future has completed 2024-11-22T03:40:20,981 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=40, resume processing ppid=39 2024-11-22T03:40:20,981 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=40, ppid=39, state=SUCCESS, hasLock=false; CloseRegionProcedure 53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 in 168 msec 2024-11-22T03:40:20,982 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=39, ppid=38, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=53a70bd4899ab7aa137c5ffbd98f8de7, REOPEN/MOVE; state=CLOSED, location=c85114ed5096,42545,1732246786482; forceNewPlan=false, retain=true 2024-11-22T03:40:21,133 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=39 updating hbase:meta row=53a70bd4899ab7aa137c5ffbd98f8de7, regionState=OPENING, regionLocation=c85114ed5096,42545,1732246786482 2024-11-22T03:40:21,135 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=39, ppid=38, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=53a70bd4899ab7aa137c5ffbd98f8de7, REOPEN/MOVE because future has completed 2024-11-22T03:40:21,135 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=41, ppid=39, state=RUNNABLE, hasLock=false; OpenRegionProcedure 53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482}] 2024-11-22T03:40:21,291 INFO [RS_OPEN_REGION-regionserver/c85114ed5096:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7. 2024-11-22T03:40:21,291 DEBUG [RS_OPEN_REGION-regionserver/c85114ed5096:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(7752): Opening region: {ENCODED => 53a70bd4899ab7aa137c5ffbd98f8de7, NAME => 'TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7.', STARTKEY => '', ENDKEY => ''} 2024-11-22T03:40:21,291 DEBUG [RS_OPEN_REGION-regionserver/c85114ed5096:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 53a70bd4899ab7aa137c5ffbd98f8de7 2024-11-22T03:40:21,291 DEBUG [RS_OPEN_REGION-regionserver/c85114ed5096:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-22T03:40:21,292 DEBUG [RS_OPEN_REGION-regionserver/c85114ed5096:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(7794): checking encryption for 53a70bd4899ab7aa137c5ffbd98f8de7 2024-11-22T03:40:21,292 DEBUG [RS_OPEN_REGION-regionserver/c85114ed5096:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(7797): checking classloading for 53a70bd4899ab7aa137c5ffbd98f8de7 2024-11-22T03:40:21,296 INFO [StoreOpener-53a70bd4899ab7aa137c5ffbd98f8de7-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 53a70bd4899ab7aa137c5ffbd98f8de7 2024-11-22T03:40:21,304 INFO [StoreOpener-53a70bd4899ab7aa137c5ffbd98f8de7-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 53a70bd4899ab7aa137c5ffbd98f8de7 columnFamilyName A 2024-11-22T03:40:21,306 DEBUG [StoreOpener-53a70bd4899ab7aa137c5ffbd98f8de7-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:21,307 INFO [StoreOpener-53a70bd4899ab7aa137c5ffbd98f8de7-1 {}] regionserver.HStore(327): Store=53a70bd4899ab7aa137c5ffbd98f8de7/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-22T03:40:21,308 INFO [StoreOpener-53a70bd4899ab7aa137c5ffbd98f8de7-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 53a70bd4899ab7aa137c5ffbd98f8de7 2024-11-22T03:40:21,309 INFO [StoreOpener-53a70bd4899ab7aa137c5ffbd98f8de7-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 53a70bd4899ab7aa137c5ffbd98f8de7 columnFamilyName B 2024-11-22T03:40:21,309 DEBUG [StoreOpener-53a70bd4899ab7aa137c5ffbd98f8de7-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:21,309 INFO [StoreOpener-53a70bd4899ab7aa137c5ffbd98f8de7-1 {}] regionserver.HStore(327): Store=53a70bd4899ab7aa137c5ffbd98f8de7/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-22T03:40:21,309 INFO [StoreOpener-53a70bd4899ab7aa137c5ffbd98f8de7-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 53a70bd4899ab7aa137c5ffbd98f8de7 2024-11-22T03:40:21,310 INFO [StoreOpener-53a70bd4899ab7aa137c5ffbd98f8de7-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 53a70bd4899ab7aa137c5ffbd98f8de7 columnFamilyName C 2024-11-22T03:40:21,310 DEBUG [StoreOpener-53a70bd4899ab7aa137c5ffbd98f8de7-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:21,311 INFO [StoreOpener-53a70bd4899ab7aa137c5ffbd98f8de7-1 {}] regionserver.HStore(327): Store=53a70bd4899ab7aa137c5ffbd98f8de7/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-22T03:40:21,311 DEBUG [RS_OPEN_REGION-regionserver/c85114ed5096:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1038): replaying wal for 53a70bd4899ab7aa137c5ffbd98f8de7 2024-11-22T03:40:21,312 DEBUG [RS_OPEN_REGION-regionserver/c85114ed5096:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7 2024-11-22T03:40:21,313 DEBUG [RS_OPEN_REGION-regionserver/c85114ed5096:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7 2024-11-22T03:40:21,314 DEBUG [RS_OPEN_REGION-regionserver/c85114ed5096:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1048): stopping wal replay for 53a70bd4899ab7aa137c5ffbd98f8de7 2024-11-22T03:40:21,314 DEBUG [RS_OPEN_REGION-regionserver/c85114ed5096:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1060): Cleaning up temporary data for 53a70bd4899ab7aa137c5ffbd98f8de7 2024-11-22T03:40:21,315 DEBUG [RS_OPEN_REGION-regionserver/c85114ed5096:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-22T03:40:21,316 DEBUG [RS_OPEN_REGION-regionserver/c85114ed5096:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1093): writing seq id for 53a70bd4899ab7aa137c5ffbd98f8de7 2024-11-22T03:40:21,317 INFO [RS_OPEN_REGION-regionserver/c85114ed5096:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1114): Opened 53a70bd4899ab7aa137c5ffbd98f8de7; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70631966, jitterRate=0.052498310804367065}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-22T03:40:21,317 DEBUG [RS_OPEN_REGION-regionserver/c85114ed5096:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 53a70bd4899ab7aa137c5ffbd98f8de7 2024-11-22T03:40:21,318 DEBUG [RS_OPEN_REGION-regionserver/c85114ed5096:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1006): Region open journal for 53a70bd4899ab7aa137c5ffbd98f8de7: Running coprocessor pre-open hook at 1732246821292Writing region info on filesystem at 1732246821292Initializing all the Stores at 1732246821293 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732246821293Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732246821293Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732246821293Cleaning up temporary data from old regions at 1732246821314 (+21 ms)Running coprocessor post-open hooks at 1732246821317 (+3 ms)Region opened successfully at 1732246821318 (+1 ms) 2024-11-22T03:40:21,319 INFO [RS_OPEN_REGION-regionserver/c85114ed5096:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., pid=41, masterSystemTime=1732246821287 2024-11-22T03:40:21,322 DEBUG [RS_OPEN_REGION-regionserver/c85114ed5096:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7. 2024-11-22T03:40:21,322 INFO [RS_OPEN_REGION-regionserver/c85114ed5096:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7. 2024-11-22T03:40:21,322 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=39 updating hbase:meta row=53a70bd4899ab7aa137c5ffbd98f8de7, regionState=OPEN, openSeqNum=5, regionLocation=c85114ed5096,42545,1732246786482 2024-11-22T03:40:21,324 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=41, ppid=39, state=RUNNABLE, hasLock=false; OpenRegionProcedure 53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 because future has completed 2024-11-22T03:40:21,327 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=41, resume processing ppid=39 2024-11-22T03:40:21,328 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=41, ppid=39, state=SUCCESS, hasLock=false; OpenRegionProcedure 53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 in 190 msec 2024-11-22T03:40:21,330 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=39, resume processing ppid=38 2024-11-22T03:40:21,330 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=39, ppid=38, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=53a70bd4899ab7aa137c5ffbd98f8de7, REOPEN/MOVE in 523 msec 2024-11-22T03:40:21,335 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=38, resume processing ppid=37 2024-11-22T03:40:21,335 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=38, ppid=37, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 537 msec 2024-11-22T03:40:21,339 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=37, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 966 msec 2024-11-22T03:40:21,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=37 2024-11-22T03:40:21,342 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-22T03:40:21,342 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@65b7b5d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T03:40:21,343 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c85114ed5096,42627,-1 for getting cluster id 2024-11-22T03:40:21,343 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-22T03:40:21,344 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '67035b7f-c649-44a8-9db7-17adff49cedb' 2024-11-22T03:40:21,344 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-22T03:40:21,344 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "67035b7f-c649-44a8-9db7-17adff49cedb" 2024-11-22T03:40:21,344 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1be4e7c7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T03:40:21,344 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c85114ed5096,42627,-1] 2024-11-22T03:40:21,345 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-22T03:40:21,345 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T03:40:21,347 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53890, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-22T03:40:21,347 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@12bcc5b7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T03:40:21,348 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6a1845b3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T03:40:21,349 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c85114ed5096,42627,-1 for getting cluster id 2024-11-22T03:40:21,349 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-22T03:40:21,350 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '67035b7f-c649-44a8-9db7-17adff49cedb' 2024-11-22T03:40:21,350 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-22T03:40:21,350 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "67035b7f-c649-44a8-9db7-17adff49cedb" 2024-11-22T03:40:21,350 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@658bd0bc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T03:40:21,350 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c85114ed5096,42627,-1] 2024-11-22T03:40:21,351 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-22T03:40:21,351 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T03:40:21,352 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53900, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-22T03:40:21,353 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2b3b53b7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T03:40:21,354 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@208b436a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T03:40:21,354 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c85114ed5096,42627,-1 for getting cluster id 2024-11-22T03:40:21,354 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-22T03:40:21,355 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '67035b7f-c649-44a8-9db7-17adff49cedb' 2024-11-22T03:40:21,356 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-22T03:40:21,356 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "67035b7f-c649-44a8-9db7-17adff49cedb" 2024-11-22T03:40:21,356 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@152e08cb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T03:40:21,356 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c85114ed5096,42627,-1] 2024-11-22T03:40:21,357 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-22T03:40:21,357 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T03:40:21,358 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53928, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-22T03:40:21,359 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@12c4f73d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T03:40:21,364 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@349085e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T03:40:21,364 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c85114ed5096,42627,-1 for getting cluster id 2024-11-22T03:40:21,365 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-22T03:40:21,366 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '67035b7f-c649-44a8-9db7-17adff49cedb' 2024-11-22T03:40:21,366 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-22T03:40:21,367 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "67035b7f-c649-44a8-9db7-17adff49cedb" 2024-11-22T03:40:21,367 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@aba95a9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T03:40:21,367 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c85114ed5096,42627,-1] 2024-11-22T03:40:21,367 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-22T03:40:21,368 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T03:40:21,369 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53952, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-22T03:40:21,369 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@f5bb40c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T03:40:21,370 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7ffb7a39, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T03:40:21,370 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c85114ed5096,42627,-1 for getting cluster id 2024-11-22T03:40:21,370 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-22T03:40:21,371 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '67035b7f-c649-44a8-9db7-17adff49cedb' 2024-11-22T03:40:21,372 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-22T03:40:21,372 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "67035b7f-c649-44a8-9db7-17adff49cedb" 2024-11-22T03:40:21,372 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@20c8c86b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T03:40:21,372 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c85114ed5096,42627,-1] 2024-11-22T03:40:21,372 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-22T03:40:21,372 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T03:40:21,373 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53970, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-22T03:40:21,374 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@22d076cf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T03:40:21,375 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3d21c225, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T03:40:21,375 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c85114ed5096,42627,-1 for getting cluster id 2024-11-22T03:40:21,376 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-22T03:40:21,376 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '67035b7f-c649-44a8-9db7-17adff49cedb' 2024-11-22T03:40:21,377 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-22T03:40:21,377 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "67035b7f-c649-44a8-9db7-17adff49cedb" 2024-11-22T03:40:21,377 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@171d4544, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T03:40:21,377 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c85114ed5096,42627,-1] 2024-11-22T03:40:21,377 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-22T03:40:21,378 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T03:40:21,379 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53996, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-22T03:40:21,379 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3ce3532, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T03:40:21,380 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5a92e196, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T03:40:21,380 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c85114ed5096,42627,-1 for getting cluster id 2024-11-22T03:40:21,380 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-22T03:40:21,381 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '67035b7f-c649-44a8-9db7-17adff49cedb' 2024-11-22T03:40:21,381 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-22T03:40:21,382 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "67035b7f-c649-44a8-9db7-17adff49cedb" 2024-11-22T03:40:21,382 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1c75e23a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T03:40:21,382 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c85114ed5096,42627,-1] 2024-11-22T03:40:21,382 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-22T03:40:21,382 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T03:40:21,383 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54024, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-22T03:40:21,384 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@35753a80, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T03:40:21,385 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3806f988, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T03:40:21,385 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c85114ed5096,42627,-1 for getting cluster id 2024-11-22T03:40:21,385 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-22T03:40:21,387 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '67035b7f-c649-44a8-9db7-17adff49cedb' 2024-11-22T03:40:21,387 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-22T03:40:21,387 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "67035b7f-c649-44a8-9db7-17adff49cedb" 2024-11-22T03:40:21,387 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@56229cca, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T03:40:21,387 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c85114ed5096,42627,-1] 2024-11-22T03:40:21,387 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-22T03:40:21,388 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T03:40:21,389 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54044, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-22T03:40:21,389 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@c5184a9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T03:40:21,390 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@494eb83f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T03:40:21,391 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c85114ed5096,42627,-1 for getting cluster id 2024-11-22T03:40:21,391 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-22T03:40:21,392 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '67035b7f-c649-44a8-9db7-17adff49cedb' 2024-11-22T03:40:21,392 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-22T03:40:21,392 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "67035b7f-c649-44a8-9db7-17adff49cedb" 2024-11-22T03:40:21,393 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4f93555f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T03:40:21,393 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c85114ed5096,42627,-1] 2024-11-22T03:40:21,393 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-22T03:40:21,393 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T03:40:21,395 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54070, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-22T03:40:21,396 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@61627e6b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T03:40:21,398 DEBUG [Thread-660 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-22T03:40:21,399 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c85114ed5096,42545,1732246786482, seqNum=-1] 2024-11-22T03:40:21,400 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-22T03:40:21,401 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34684, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-22T03:40:21,405 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5] 2024-11-22T03:40:21,405 DEBUG [Thread-677 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-22T03:40:21,407 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c85114ed5096,42545,1732246786482, seqNum=-1] 2024-11-22T03:40:21,407 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-22T03:40:21,409 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34694, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-22T03:40:21,413 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5] 2024-11-22T03:40:21,422 DEBUG [Thread-673 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-22T03:40:21,424 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c85114ed5096,42545,1732246786482, seqNum=-1] 2024-11-22T03:40:21,425 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-22T03:40:21,427 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34710, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-22T03:40:21,430 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5] 2024-11-22T03:40:21,437 DEBUG [Thread-675 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-22T03:40:21,438 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c85114ed5096,42545,1732246786482, seqNum=-1] 2024-11-22T03:40:21,439 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-22T03:40:21,442 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34720, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-22T03:40:21,444 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5] 2024-11-22T03:40:21,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(8855): Flush requested on 53a70bd4899ab7aa137c5ffbd98f8de7 2024-11-22T03:40:21,453 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-22T03:40:21,453 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 53a70bd4899ab7aa137c5ffbd98f8de7 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-22T03:40:21,457 DEBUG [Thread-668 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-22T03:40:21,457 DEBUG [Thread-671 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-22T03:40:21,458 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c85114ed5096,42545,1732246786482, seqNum=-1] 2024-11-22T03:40:21,458 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c85114ed5096,42545,1732246786482, seqNum=-1] 2024-11-22T03:40:21,458 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-22T03:40:21,458 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-22T03:40:21,460 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34730, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-22T03:40:21,460 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34728, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-22T03:40:21,464 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5] 2024-11-22T03:40:21,464 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5] 2024-11-22T03:40:21,471 DEBUG [Thread-666 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-22T03:40:21,472 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c85114ed5096,42545,1732246786482, seqNum=-1] 2024-11-22T03:40:21,472 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-22T03:40:21,474 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34746, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-22T03:40:21,477 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5] 2024-11-22T03:40:21,480 DEBUG [Thread-664 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-22T03:40:21,481 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c85114ed5096,42545,1732246786482, seqNum=-1] 2024-11-22T03:40:21,482 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-22T03:40:21,483 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34754, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-22T03:40:21,485 DEBUG [Thread-662 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-22T03:40:21,486 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c85114ed5096,42545,1732246786482, seqNum=-1] 2024-11-22T03:40:21,487 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-22T03:40:21,488 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34770, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-22T03:40:21,496 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42627 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-22T03:40:21,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42627 {}] procedure2.ProcedureExecutor(1139): Stored pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees 2024-11-22T03:40:21,500 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-22T03:40:21,501 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-22T03:40:21,505 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5] 2024-11-22T03:40:21,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-22T03:40:21,505 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5] 2024-11-22T03:40:21,506 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=43, ppid=42, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-22T03:40:21,550 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411229748ceaee6bd49abb1f4c678ddb5f0fb_53a70bd4899ab7aa137c5ffbd98f8de7 is 50, key is test_row_0/A:col10/1732246821451/Put/seqid=0 2024-11-22T03:40:21,582 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:21,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 4 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34754 deadline: 1732246881567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:21,584 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:21,584 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:21,584 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:40:21,584 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:21,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 4 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34770 deadline: 1732246881568, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:21,585 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:21,585 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:21,585 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:40:21,585 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:21,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34728 deadline: 1732246881573, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:21,586 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:21,586 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:21,586 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:40:21,595 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:21,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34684 deadline: 1732246881580, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:21,595 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:21,596 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:21,596 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:40:21,596 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:21,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34746 deadline: 1732246881583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:21,597 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:21,597 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:21,597 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:40:21,599 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073741957_1133 (size=17034) 2024-11-22T03:40:21,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-22T03:40:21,660 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42545 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-22T03:40:21,660 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7. 2024-11-22T03:40:21,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7. as already flushing 2024-11-22T03:40:21,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7. 2024-11-22T03:40:21,661 ERROR [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 53a70bd4899ab7aa137c5ffbd98f8de7, NAME => 'TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:40:21,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 53a70bd4899ab7aa137c5ffbd98f8de7, NAME => 'TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:40:21,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42627 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 53a70bd4899ab7aa137c5ffbd98f8de7, NAME => 'TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 53a70bd4899ab7aa137c5ffbd98f8de7, NAME => 'TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:40:21,690 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:21,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34770 deadline: 1732246881689, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:21,691 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:21,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34754 deadline: 1732246881689, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:21,691 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:21,692 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:21,692 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:21,692 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:40:21,692 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:21,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34728 deadline: 1732246881689, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:21,692 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:21,692 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:40:21,693 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:21,693 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:21,693 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:40:21,705 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:21,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34684 deadline: 1732246881699, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:21,708 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:21,708 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:21,708 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:40:21,711 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:21,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34746 deadline: 1732246881709, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:21,712 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:21,712 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:21,712 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:40:21,753 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-22T03:40:21,815 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42545 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-22T03:40:21,816 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7. 2024-11-22T03:40:21,816 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7. as already flushing 2024-11-22T03:40:21,816 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7. 2024-11-22T03:40:21,816 ERROR [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 53a70bd4899ab7aa137c5ffbd98f8de7, NAME => 'TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:40:21,816 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 53a70bd4899ab7aa137c5ffbd98f8de7, NAME => 'TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:40:21,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42627 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 53a70bd4899ab7aa137c5ffbd98f8de7, NAME => 'TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 53a70bd4899ab7aa137c5ffbd98f8de7, NAME => 'TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:40:21,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-22T03:40:21,907 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:21,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34754 deadline: 1732246881899, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:21,909 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:21,909 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:21,909 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:40:21,909 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:21,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34728 deadline: 1732246881899, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:21,910 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:21,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34770 deadline: 1732246881899, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:21,911 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:21,911 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:21,911 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:21,911 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:21,911 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:40:21,911 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:40:21,930 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:21,930 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:21,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34684 deadline: 1732246881919, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:21,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34746 deadline: 1732246881919, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:21,931 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:21,931 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:21,931 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:21,931 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:40:21,931 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:21,931 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:40:21,970 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42545 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-22T03:40:21,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7. 2024-11-22T03:40:21,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7. as already flushing 2024-11-22T03:40:21,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7. 2024-11-22T03:40:21,971 ERROR [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 53a70bd4899ab7aa137c5ffbd98f8de7, NAME => 'TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:40:21,971 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 53a70bd4899ab7aa137c5ffbd98f8de7, NAME => 'TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:40:21,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42627 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 53a70bd4899ab7aa137c5ffbd98f8de7, NAME => 'TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 53a70bd4899ab7aa137c5ffbd98f8de7, NAME => 'TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:40:22,001 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:22,007 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411229748ceaee6bd49abb1f4c678ddb5f0fb_53a70bd4899ab7aa137c5ffbd98f8de7 to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411229748ceaee6bd49abb1f4c678ddb5f0fb_53a70bd4899ab7aa137c5ffbd98f8de7 2024-11-22T03:40:22,011 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/.tmp/A/adbcfb9d053f42d09e39c8f01d7386e5, store: [table=TestAcidGuarantees family=A region=53a70bd4899ab7aa137c5ffbd98f8de7] 2024-11-22T03:40:22,019 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/.tmp/A/adbcfb9d053f42d09e39c8f01d7386e5 is 175, key is test_row_0/A:col10/1732246821451/Put/seqid=0 2024-11-22T03:40:22,059 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073741958_1134 (size=48139) 2024-11-22T03:40:22,063 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=15, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/.tmp/A/adbcfb9d053f42d09e39c8f01d7386e5 2024-11-22T03:40:22,102 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/.tmp/B/78300911a140463dbffb11243c5aac63 is 50, key is test_row_0/B:col10/1732246821451/Put/seqid=0 2024-11-22T03:40:22,124 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42545 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-22T03:40:22,124 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7. 2024-11-22T03:40:22,124 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7. as already flushing 2024-11-22T03:40:22,124 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7. 2024-11-22T03:40:22,124 ERROR [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 53a70bd4899ab7aa137c5ffbd98f8de7, NAME => 'TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:40:22,124 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 53a70bd4899ab7aa137c5ffbd98f8de7, NAME => 'TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:40:22,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42627 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 53a70bd4899ab7aa137c5ffbd98f8de7, NAME => 'TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 53a70bd4899ab7aa137c5ffbd98f8de7, NAME => 'TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:40:22,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-22T03:40:22,148 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073741959_1135 (size=12001) 2024-11-22T03:40:22,224 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:22,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34754 deadline: 1732246882218, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:22,225 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:22,224 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:22,225 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:22,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34728 deadline: 1732246882219, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:22,225 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:40:22,225 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:22,225 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:22,225 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:22,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34770 deadline: 1732246882219, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:22,225 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:40:22,226 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:22,226 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:22,226 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:40:22,240 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:22,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34746 deadline: 1732246882240, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:22,241 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:22,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34684 deadline: 1732246882240, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:22,241 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:22,241 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:22,241 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:40:22,241 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:22,241 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:22,242 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:40:22,278 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42545 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-22T03:40:22,278 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7. 2024-11-22T03:40:22,278 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7. as already flushing 2024-11-22T03:40:22,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7. 2024-11-22T03:40:22,279 ERROR [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 53a70bd4899ab7aa137c5ffbd98f8de7, NAME => 'TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:40:22,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 53a70bd4899ab7aa137c5ffbd98f8de7, NAME => 'TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:40:22,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42627 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 53a70bd4899ab7aa137c5ffbd98f8de7, NAME => 'TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 53a70bd4899ab7aa137c5ffbd98f8de7, NAME => 'TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:40:22,433 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42545 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-22T03:40:22,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7. 2024-11-22T03:40:22,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7. as already flushing 2024-11-22T03:40:22,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7. 2024-11-22T03:40:22,434 ERROR [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 53a70bd4899ab7aa137c5ffbd98f8de7, NAME => 'TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:40:22,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 53a70bd4899ab7aa137c5ffbd98f8de7, NAME => 'TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:40:22,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42627 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 53a70bd4899ab7aa137c5ffbd98f8de7, NAME => 'TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 53a70bd4899ab7aa137c5ffbd98f8de7, NAME => 'TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:40:22,548 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/.tmp/B/78300911a140463dbffb11243c5aac63 2024-11-22T03:40:22,577 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/.tmp/C/23755d88fe7a49a39fab68586618038f is 50, key is test_row_0/C:col10/1732246821451/Put/seqid=0 2024-11-22T03:40:22,589 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42545 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-22T03:40:22,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7. 2024-11-22T03:40:22,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7. as already flushing 2024-11-22T03:40:22,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7. 2024-11-22T03:40:22,590 ERROR [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 53a70bd4899ab7aa137c5ffbd98f8de7, NAME => 'TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:40:22,590 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 53a70bd4899ab7aa137c5ffbd98f8de7, NAME => 'TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:40:22,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42627 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 53a70bd4899ab7aa137c5ffbd98f8de7, NAME => 'TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 53a70bd4899ab7aa137c5ffbd98f8de7, NAME => 'TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:40:22,610 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073741960_1136 (size=12001) 2024-11-22T03:40:22,612 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/.tmp/C/23755d88fe7a49a39fab68586618038f 2024-11-22T03:40:22,625 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/.tmp/A/adbcfb9d053f42d09e39c8f01d7386e5 as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/A/adbcfb9d053f42d09e39c8f01d7386e5 2024-11-22T03:40:22,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-22T03:40:22,641 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/A/adbcfb9d053f42d09e39c8f01d7386e5, entries=250, sequenceid=15, filesize=47.0 K 2024-11-22T03:40:22,645 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/.tmp/B/78300911a140463dbffb11243c5aac63 as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/B/78300911a140463dbffb11243c5aac63 2024-11-22T03:40:22,666 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/B/78300911a140463dbffb11243c5aac63, entries=150, sequenceid=15, filesize=11.7 K 2024-11-22T03:40:22,669 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/.tmp/C/23755d88fe7a49a39fab68586618038f as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/C/23755d88fe7a49a39fab68586618038f 2024-11-22T03:40:22,680 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/C/23755d88fe7a49a39fab68586618038f, entries=150, sequenceid=15, filesize=11.7 K 2024-11-22T03:40:22,681 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 53a70bd4899ab7aa137c5ffbd98f8de7 in 1228ms, sequenceid=15, compaction requested=false 2024-11-22T03:40:22,681 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-22T03:40:22,682 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 53a70bd4899ab7aa137c5ffbd98f8de7: 2024-11-22T03:40:22,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(8855): Flush requested on 53a70bd4899ab7aa137c5ffbd98f8de7 2024-11-22T03:40:22,740 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-22T03:40:22,740 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 53a70bd4899ab7aa137c5ffbd98f8de7 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-22T03:40:22,748 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42545 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-22T03:40:22,748 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7. 2024-11-22T03:40:22,748 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7. as already flushing 2024-11-22T03:40:22,748 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7. 2024-11-22T03:40:22,748 ERROR [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 53a70bd4899ab7aa137c5ffbd98f8de7, NAME => 'TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:40:22,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 53a70bd4899ab7aa137c5ffbd98f8de7, NAME => 'TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:40:22,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42627 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 53a70bd4899ab7aa137c5ffbd98f8de7, NAME => 'TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 53a70bd4899ab7aa137c5ffbd98f8de7, NAME => 'TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:40:22,772 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:22,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34728 deadline: 1732246882755, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:22,773 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:22,773 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:22,773 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:40:22,779 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:22,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34754 deadline: 1732246882769, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:22,780 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:22,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34746 deadline: 1732246882770, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:22,781 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:22,781 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:22,781 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:40:22,781 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:22,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34684 deadline: 1732246882773, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:22,782 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:22,782 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:22,782 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:22,782 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:40:22,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34770 deadline: 1732246882774, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:22,784 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:22,784 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:22,784 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:40:22,784 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:22,784 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:22,784 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:40:22,785 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241122e5d254e71603421aabcc41d74b4d41e4_53a70bd4899ab7aa137c5ffbd98f8de7 is 50, key is test_row_0/A:col10/1732246821577/Put/seqid=0 2024-11-22T03:40:22,847 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073741961_1137 (size=14594) 2024-11-22T03:40:22,885 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:22,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34728 deadline: 1732246882879, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:22,887 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:22,887 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:22,887 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:40:22,892 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:22,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34746 deadline: 1732246882890, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:22,897 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:22,898 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:22,898 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:40:22,898 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:22,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34754 deadline: 1732246882890, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:22,899 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:22,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34770 deadline: 1732246882891, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:22,900 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:22,900 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:22,900 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:22,900 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:22,900 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:40:22,900 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:40:22,904 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42545 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-22T03:40:22,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7. 2024-11-22T03:40:22,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7. as already flushing 2024-11-22T03:40:22,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7. 2024-11-22T03:40:22,905 ERROR [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 53a70bd4899ab7aa137c5ffbd98f8de7, NAME => 'TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:40:22,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 53a70bd4899ab7aa137c5ffbd98f8de7, NAME => 'TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:40:22,905 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:22,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34684 deadline: 1732246882890, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:22,906 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:22,906 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:22,906 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:40:22,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42627 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 53a70bd4899ab7aa137c5ffbd98f8de7, NAME => 'TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 53a70bd4899ab7aa137c5ffbd98f8de7, NAME => 'TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:40:23,060 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42545 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-22T03:40:23,061 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7. 2024-11-22T03:40:23,061 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7. as already flushing 2024-11-22T03:40:23,061 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7. 2024-11-22T03:40:23,061 ERROR [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 53a70bd4899ab7aa137c5ffbd98f8de7, NAME => 'TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:40:23,061 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 53a70bd4899ab7aa137c5ffbd98f8de7, NAME => 'TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:40:23,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42627 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 53a70bd4899ab7aa137c5ffbd98f8de7, NAME => 'TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 53a70bd4899ab7aa137c5ffbd98f8de7, NAME => 'TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:40:23,102 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:23,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34728 deadline: 1732246883098, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:23,103 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:23,103 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:23,103 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:40:23,115 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:23,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34770 deadline: 1732246883109, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:23,116 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:23,116 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:23,116 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:23,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34754 deadline: 1732246883110, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:23,116 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:40:23,116 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:23,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34746 deadline: 1732246883110, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:23,117 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:23,117 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:23,117 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:40:23,117 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:23,117 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:23,117 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:40:23,121 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:23,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34684 deadline: 1732246883119, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:23,121 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:23,121 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:23,122 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:40:23,215 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42545 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-22T03:40:23,216 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7. 2024-11-22T03:40:23,216 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7. as already flushing 2024-11-22T03:40:23,216 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7. 2024-11-22T03:40:23,216 ERROR [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 53a70bd4899ab7aa137c5ffbd98f8de7, NAME => 'TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:40:23,216 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 53a70bd4899ab7aa137c5ffbd98f8de7, NAME => 'TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:40:23,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42627 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 53a70bd4899ab7aa137c5ffbd98f8de7, NAME => 'TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 53a70bd4899ab7aa137c5ffbd98f8de7, NAME => 'TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:40:23,248 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:23,254 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241122e5d254e71603421aabcc41d74b4d41e4_53a70bd4899ab7aa137c5ffbd98f8de7 to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241122e5d254e71603421aabcc41d74b4d41e4_53a70bd4899ab7aa137c5ffbd98f8de7 2024-11-22T03:40:23,256 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/.tmp/A/e5deced64e254ca1990394b04a60031b, store: [table=TestAcidGuarantees family=A region=53a70bd4899ab7aa137c5ffbd98f8de7] 2024-11-22T03:40:23,256 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/.tmp/A/e5deced64e254ca1990394b04a60031b is 175, key is test_row_0/A:col10/1732246821577/Put/seqid=0 2024-11-22T03:40:23,263 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073741962_1138 (size=39549) 2024-11-22T03:40:23,370 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42545 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-22T03:40:23,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7. 2024-11-22T03:40:23,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7. as already flushing 2024-11-22T03:40:23,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7. 2024-11-22T03:40:23,370 ERROR [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 53a70bd4899ab7aa137c5ffbd98f8de7, NAME => 'TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:40:23,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 53a70bd4899ab7aa137c5ffbd98f8de7, NAME => 'TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:40:23,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42627 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 53a70bd4899ab7aa137c5ffbd98f8de7, NAME => 'TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 53a70bd4899ab7aa137c5ffbd98f8de7, NAME => 'TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:40:23,411 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:23,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34728 deadline: 1732246883409, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:23,412 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:23,412 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:23,412 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:40:23,430 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:23,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34684 deadline: 1732246883429, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:23,431 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:23,431 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:23,431 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:40:23,431 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:23,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34746 deadline: 1732246883429, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:23,432 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:23,432 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:23,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34770 deadline: 1732246883429, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:23,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34754 deadline: 1732246883429, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:23,432 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:23,432 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:23,433 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:23,433 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:23,433 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:40:23,433 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:40:23,433 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:23,433 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:23,433 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:40:23,524 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42545 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-22T03:40:23,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7. 2024-11-22T03:40:23,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7. as already flushing 2024-11-22T03:40:23,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7. 2024-11-22T03:40:23,524 ERROR [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 53a70bd4899ab7aa137c5ffbd98f8de7, NAME => 'TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:40:23,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 53a70bd4899ab7aa137c5ffbd98f8de7, NAME => 'TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:40:23,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42627 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 53a70bd4899ab7aa137c5ffbd98f8de7, NAME => 'TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 53a70bd4899ab7aa137c5ffbd98f8de7, NAME => 'TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:40:23,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-22T03:40:23,666 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=41, memsize=51.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/.tmp/A/e5deced64e254ca1990394b04a60031b 2024-11-22T03:40:23,679 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42545 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-22T03:40:23,679 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7. 2024-11-22T03:40:23,679 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7. as already flushing 2024-11-22T03:40:23,679 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7. 2024-11-22T03:40:23,679 ERROR [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 53a70bd4899ab7aa137c5ffbd98f8de7, NAME => 'TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:40:23,679 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 53a70bd4899ab7aa137c5ffbd98f8de7, NAME => 'TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:40:23,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42627 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 53a70bd4899ab7aa137c5ffbd98f8de7, NAME => 'TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 53a70bd4899ab7aa137c5ffbd98f8de7, NAME => 'TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:40:23,682 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/.tmp/B/6e91529c14514ad7a28fdc83c85bec71 is 50, key is test_row_0/B:col10/1732246821577/Put/seqid=0 2024-11-22T03:40:23,726 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073741963_1139 (size=12001) 2024-11-22T03:40:23,727 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=41 (bloomFilter=true), to=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/.tmp/B/6e91529c14514ad7a28fdc83c85bec71 2024-11-22T03:40:23,752 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/.tmp/C/4305809f7be94277adba899314ed55e5 is 50, key is test_row_0/C:col10/1732246821577/Put/seqid=0 2024-11-22T03:40:23,811 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073741964_1140 (size=12001) 2024-11-22T03:40:23,812 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=41 (bloomFilter=true), to=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/.tmp/C/4305809f7be94277adba899314ed55e5 2024-11-22T03:40:23,821 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/.tmp/A/e5deced64e254ca1990394b04a60031b as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/A/e5deced64e254ca1990394b04a60031b 2024-11-22T03:40:23,830 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/A/e5deced64e254ca1990394b04a60031b, entries=200, sequenceid=41, filesize=38.6 K 2024-11-22T03:40:23,832 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/.tmp/B/6e91529c14514ad7a28fdc83c85bec71 as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/B/6e91529c14514ad7a28fdc83c85bec71 2024-11-22T03:40:23,833 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42545 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-22T03:40:23,833 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7. 2024-11-22T03:40:23,833 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7. as already flushing 2024-11-22T03:40:23,833 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7. 2024-11-22T03:40:23,834 ERROR [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 53a70bd4899ab7aa137c5ffbd98f8de7, NAME => 'TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:40:23,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 53a70bd4899ab7aa137c5ffbd98f8de7, NAME => 'TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:40:23,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42627 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 53a70bd4899ab7aa137c5ffbd98f8de7, NAME => 'TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 53a70bd4899ab7aa137c5ffbd98f8de7, NAME => 'TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:40:23,856 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/B/6e91529c14514ad7a28fdc83c85bec71, entries=150, sequenceid=41, filesize=11.7 K 2024-11-22T03:40:23,859 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/.tmp/C/4305809f7be94277adba899314ed55e5 as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/C/4305809f7be94277adba899314ed55e5 2024-11-22T03:40:23,873 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/C/4305809f7be94277adba899314ed55e5, entries=150, sequenceid=41, filesize=11.7 K 2024-11-22T03:40:23,874 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for 53a70bd4899ab7aa137c5ffbd98f8de7 in 1134ms, sequenceid=41, compaction requested=false 2024-11-22T03:40:23,874 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 53a70bd4899ab7aa137c5ffbd98f8de7: 2024-11-22T03:40:23,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(8855): Flush requested on 53a70bd4899ab7aa137c5ffbd98f8de7 2024-11-22T03:40:23,932 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-22T03:40:23,932 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 53a70bd4899ab7aa137c5ffbd98f8de7 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-22T03:40:23,961 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411228748b81d00cb412aa945bc57a31502db_53a70bd4899ab7aa137c5ffbd98f8de7 is 50, key is test_row_0/A:col10/1732246822768/Put/seqid=0 2024-11-22T03:40:23,968 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073741965_1141 (size=14594) 2024-11-22T03:40:23,970 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:23,979 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411228748b81d00cb412aa945bc57a31502db_53a70bd4899ab7aa137c5ffbd98f8de7 to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411228748b81d00cb412aa945bc57a31502db_53a70bd4899ab7aa137c5ffbd98f8de7 2024-11-22T03:40:23,981 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/.tmp/A/d8349d7cb4304958b4fc704955f33898, store: [table=TestAcidGuarantees family=A region=53a70bd4899ab7aa137c5ffbd98f8de7] 2024-11-22T03:40:23,982 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/.tmp/A/d8349d7cb4304958b4fc704955f33898 is 175, key is test_row_0/A:col10/1732246822768/Put/seqid=0 2024-11-22T03:40:23,988 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42545 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-22T03:40:23,989 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7. 2024-11-22T03:40:23,989 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7. as already flushing 2024-11-22T03:40:23,989 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7. 2024-11-22T03:40:23,989 ERROR [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 53a70bd4899ab7aa137c5ffbd98f8de7, NAME => 'TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:40:23,989 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 53a70bd4899ab7aa137c5ffbd98f8de7, NAME => 'TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:40:23,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42627 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 53a70bd4899ab7aa137c5ffbd98f8de7, NAME => 'TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 53a70bd4899ab7aa137c5ffbd98f8de7, NAME => 'TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:40:23,991 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073741966_1142 (size=39549) 2024-11-22T03:40:23,991 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:23,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34754 deadline: 1732246883985, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:23,992 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:23,992 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:23,992 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:40:23,992 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=52, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/.tmp/A/d8349d7cb4304958b4fc704955f33898 2024-11-22T03:40:23,995 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:23,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34746 deadline: 1732246883986, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:23,995 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:23,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34684 deadline: 1732246883986, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:23,995 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:23,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34728 deadline: 1732246883987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:23,996 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:23,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34770 deadline: 1732246883988, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:23,996 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:23,996 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:23,996 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:23,996 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:40:23,996 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:23,996 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:40:23,996 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:23,997 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:23,997 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:40:23,997 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:23,997 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:23,997 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:40:24,010 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/.tmp/B/5591c9c3e4004cbaac9e8867ef88ff7c is 50, key is test_row_0/B:col10/1732246822768/Put/seqid=0 2024-11-22T03:40:24,042 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073741967_1143 (size=12001) 2024-11-22T03:40:24,043 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=52 (bloomFilter=true), to=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/.tmp/B/5591c9c3e4004cbaac9e8867ef88ff7c 2024-11-22T03:40:24,067 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/.tmp/C/7f3b2c7d4c3946629603539a7af7692f is 50, key is test_row_0/C:col10/1732246822768/Put/seqid=0 2024-11-22T03:40:24,075 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073741968_1144 (size=12001) 2024-11-22T03:40:24,100 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:24,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34754 deadline: 1732246884098, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:24,101 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:24,101 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:24,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34684 deadline: 1732246884099, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:24,101 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:24,101 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:40:24,101 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:24,101 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:24,102 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:40:24,102 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:24,102 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:24,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34770 deadline: 1732246884099, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:24,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34728 deadline: 1732246884099, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:24,102 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:24,102 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:24,102 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:40:24,102 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:24,102 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:24,102 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:40:24,114 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:24,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34746 deadline: 1732246884109, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:24,116 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:24,116 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:24,117 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:40:24,144 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42545 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-22T03:40:24,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7. 2024-11-22T03:40:24,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7. as already flushing 2024-11-22T03:40:24,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7. 2024-11-22T03:40:24,144 ERROR [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 53a70bd4899ab7aa137c5ffbd98f8de7, NAME => 'TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:40:24,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 53a70bd4899ab7aa137c5ffbd98f8de7, NAME => 'TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:40:24,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42627 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 53a70bd4899ab7aa137c5ffbd98f8de7, NAME => 'TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 53a70bd4899ab7aa137c5ffbd98f8de7, NAME => 'TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:40:24,297 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42545 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-22T03:40:24,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7. 2024-11-22T03:40:24,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7. as already flushing 2024-11-22T03:40:24,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7. 2024-11-22T03:40:24,298 ERROR [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 53a70bd4899ab7aa137c5ffbd98f8de7, NAME => 'TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:40:24,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 53a70bd4899ab7aa137c5ffbd98f8de7, NAME => 'TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:40:24,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42627 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 53a70bd4899ab7aa137c5ffbd98f8de7, NAME => 'TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 53a70bd4899ab7aa137c5ffbd98f8de7, NAME => 'TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:40:24,310 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:24,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34754 deadline: 1732246884308, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:24,310 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:24,311 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:24,311 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:24,311 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:40:24,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34684 deadline: 1732246884309, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:24,311 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:24,311 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:24,311 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:24,311 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:40:24,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34770 deadline: 1732246884309, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:24,311 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:24,311 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:24,311 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:24,311 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:40:24,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34728 deadline: 1732246884309, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:24,312 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:24,312 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:24,312 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:40:24,320 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:24,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34746 deadline: 1732246884318, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:24,321 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:24,321 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:24,321 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:40:24,451 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42545 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-22T03:40:24,451 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7. 2024-11-22T03:40:24,451 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7. as already flushing 2024-11-22T03:40:24,451 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7. 2024-11-22T03:40:24,451 ERROR [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 53a70bd4899ab7aa137c5ffbd98f8de7, NAME => 'TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:40:24,451 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 53a70bd4899ab7aa137c5ffbd98f8de7, NAME => 'TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:40:24,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42627 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 53a70bd4899ab7aa137c5ffbd98f8de7, NAME => 'TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 53a70bd4899ab7aa137c5ffbd98f8de7, NAME => 'TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:40:24,477 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=52 (bloomFilter=true), to=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/.tmp/C/7f3b2c7d4c3946629603539a7af7692f 2024-11-22T03:40:24,486 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/.tmp/A/d8349d7cb4304958b4fc704955f33898 as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/A/d8349d7cb4304958b4fc704955f33898 2024-11-22T03:40:24,494 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/A/d8349d7cb4304958b4fc704955f33898, entries=200, sequenceid=52, filesize=38.6 K 2024-11-22T03:40:24,495 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/.tmp/B/5591c9c3e4004cbaac9e8867ef88ff7c as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/B/5591c9c3e4004cbaac9e8867ef88ff7c 2024-11-22T03:40:24,505 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/B/5591c9c3e4004cbaac9e8867ef88ff7c, entries=150, sequenceid=52, filesize=11.7 K 2024-11-22T03:40:24,506 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/.tmp/C/7f3b2c7d4c3946629603539a7af7692f as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/C/7f3b2c7d4c3946629603539a7af7692f 2024-11-22T03:40:24,516 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/C/7f3b2c7d4c3946629603539a7af7692f, entries=150, sequenceid=52, filesize=11.7 K 2024-11-22T03:40:24,518 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 53a70bd4899ab7aa137c5ffbd98f8de7 in 585ms, sequenceid=52, compaction requested=true 2024-11-22T03:40:24,518 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 53a70bd4899ab7aa137c5ffbd98f8de7: 2024-11-22T03:40:24,518 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 53a70bd4899ab7aa137c5ffbd98f8de7:A, priority=-2147483648, current under compaction store size is 1 2024-11-22T03:40:24,518 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T03:40:24,518 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T03:40:24,518 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 53a70bd4899ab7aa137c5ffbd98f8de7:B, priority=-2147483648, current under compaction store size is 2 2024-11-22T03:40:24,518 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T03:40:24,518 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T03:40:24,518 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 53a70bd4899ab7aa137c5ffbd98f8de7:C, priority=-2147483648, current under compaction store size is 3 2024-11-22T03:40:24,518 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T03:40:24,519 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T03:40:24,519 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 127237 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T03:40:24,519 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HStore(1541): 53a70bd4899ab7aa137c5ffbd98f8de7/B is initiating minor compaction (all files) 2024-11-22T03:40:24,519 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.HStore(1541): 53a70bd4899ab7aa137c5ffbd98f8de7/A is initiating minor compaction (all files) 2024-11-22T03:40:24,520 INFO [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 53a70bd4899ab7aa137c5ffbd98f8de7/A in TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7. 2024-11-22T03:40:24,520 INFO [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 53a70bd4899ab7aa137c5ffbd98f8de7/B in TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7. 2024-11-22T03:40:24,520 INFO [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/B/78300911a140463dbffb11243c5aac63, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/B/6e91529c14514ad7a28fdc83c85bec71, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/B/5591c9c3e4004cbaac9e8867ef88ff7c] into tmpdir=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/.tmp, totalSize=35.2 K 2024-11-22T03:40:24,520 INFO [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/A/adbcfb9d053f42d09e39c8f01d7386e5, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/A/e5deced64e254ca1990394b04a60031b, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/A/d8349d7cb4304958b4fc704955f33898] into tmpdir=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/.tmp, totalSize=124.3 K 2024-11-22T03:40:24,520 INFO [RS:0;c85114ed5096:42545-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7. 2024-11-22T03:40:24,521 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] compactions.Compactor(225): Compacting 78300911a140463dbffb11243c5aac63, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1732246821436 2024-11-22T03:40:24,520 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7. files: [hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/A/adbcfb9d053f42d09e39c8f01d7386e5, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/A/e5deced64e254ca1990394b04a60031b, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/A/d8349d7cb4304958b4fc704955f33898] 2024-11-22T03:40:24,521 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] compactions.Compactor(225): Compacting 6e91529c14514ad7a28fdc83c85bec71, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=41, earliestPutTs=1732246821568 2024-11-22T03:40:24,521 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] compactions.Compactor(225): Compacting adbcfb9d053f42d09e39c8f01d7386e5, keycount=250, bloomtype=ROW, size=47.0 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1732246821412 2024-11-22T03:40:24,521 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] compactions.Compactor(225): Compacting 5591c9c3e4004cbaac9e8867ef88ff7c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1732246822763 2024-11-22T03:40:24,522 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] compactions.Compactor(225): Compacting e5deced64e254ca1990394b04a60031b, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=41, earliestPutTs=1732246821568 2024-11-22T03:40:24,522 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] compactions.Compactor(225): Compacting d8349d7cb4304958b4fc704955f33898, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1732246822763 2024-11-22T03:40:24,533 INFO [RS:0;c85114ed5096:42545-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=53a70bd4899ab7aa137c5ffbd98f8de7] 2024-11-22T03:40:24,535 INFO [RS:0;c85114ed5096:42545-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 53a70bd4899ab7aa137c5ffbd98f8de7#B#compaction#126 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-22T03:40:24,536 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/.tmp/B/0fe94b926f4b4fa2a0d02cc23e244525 is 50, key is test_row_0/B:col10/1732246822768/Put/seqid=0 2024-11-22T03:40:24,538 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411228af7823ae851413380f0e49987206333_53a70bd4899ab7aa137c5ffbd98f8de7 store=[table=TestAcidGuarantees family=A region=53a70bd4899ab7aa137c5ffbd98f8de7] 2024-11-22T03:40:24,547 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411228af7823ae851413380f0e49987206333_53a70bd4899ab7aa137c5ffbd98f8de7, store=[table=TestAcidGuarantees family=A region=53a70bd4899ab7aa137c5ffbd98f8de7] 2024-11-22T03:40:24,547 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411228af7823ae851413380f0e49987206333_53a70bd4899ab7aa137c5ffbd98f8de7 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=53a70bd4899ab7aa137c5ffbd98f8de7] 2024-11-22T03:40:24,561 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073741969_1145 (size=12104) 2024-11-22T03:40:24,568 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/.tmp/B/0fe94b926f4b4fa2a0d02cc23e244525 as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/B/0fe94b926f4b4fa2a0d02cc23e244525 2024-11-22T03:40:24,576 INFO [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 53a70bd4899ab7aa137c5ffbd98f8de7/B of 53a70bd4899ab7aa137c5ffbd98f8de7 into 0fe94b926f4b4fa2a0d02cc23e244525(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T03:40:24,576 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 53a70bd4899ab7aa137c5ffbd98f8de7: 2024-11-22T03:40:24,576 INFO [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., storeName=53a70bd4899ab7aa137c5ffbd98f8de7/B, priority=13, startTime=1732246824518; duration=0sec 2024-11-22T03:40:24,576 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T03:40:24,576 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 53a70bd4899ab7aa137c5ffbd98f8de7:B 2024-11-22T03:40:24,577 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T03:40:24,579 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T03:40:24,580 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HStore(1541): 53a70bd4899ab7aa137c5ffbd98f8de7/C is initiating minor compaction (all files) 2024-11-22T03:40:24,580 INFO [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 53a70bd4899ab7aa137c5ffbd98f8de7/C in TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7. 2024-11-22T03:40:24,580 INFO [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/C/23755d88fe7a49a39fab68586618038f, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/C/4305809f7be94277adba899314ed55e5, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/C/7f3b2c7d4c3946629603539a7af7692f] into tmpdir=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/.tmp, totalSize=35.2 K 2024-11-22T03:40:24,580 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] compactions.Compactor(225): Compacting 23755d88fe7a49a39fab68586618038f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1732246821436 2024-11-22T03:40:24,582 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] compactions.Compactor(225): Compacting 4305809f7be94277adba899314ed55e5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=41, earliestPutTs=1732246821568 2024-11-22T03:40:24,582 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073741970_1146 (size=4469) 2024-11-22T03:40:24,582 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] compactions.Compactor(225): Compacting 7f3b2c7d4c3946629603539a7af7692f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1732246822763 2024-11-22T03:40:24,583 INFO [RS:0;c85114ed5096:42545-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 53a70bd4899ab7aa137c5ffbd98f8de7#A#compaction#127 average throughput is 0.49 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T03:40:24,584 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/.tmp/A/11fcb99717c4471d9f90041c22e98b38 is 175, key is test_row_0/A:col10/1732246822768/Put/seqid=0 2024-11-22T03:40:24,602 INFO [RS:0;c85114ed5096:42545-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 53a70bd4899ab7aa137c5ffbd98f8de7#C#compaction#128 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T03:40:24,602 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/.tmp/C/f935efd1741a4353b24f1b2e16390aa2 is 50, key is test_row_0/C:col10/1732246822768/Put/seqid=0 2024-11-22T03:40:24,606 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073741971_1147 (size=31058) 2024-11-22T03:40:24,609 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42545 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-22T03:40:24,609 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7. 2024-11-22T03:40:24,610 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2902): Flushing 53a70bd4899ab7aa137c5ffbd98f8de7 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-22T03:40:24,616 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/.tmp/A/11fcb99717c4471d9f90041c22e98b38 as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/A/11fcb99717c4471d9f90041c22e98b38 2024-11-22T03:40:24,623 INFO [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 53a70bd4899ab7aa137c5ffbd98f8de7/A of 53a70bd4899ab7aa137c5ffbd98f8de7 into 11fcb99717c4471d9f90041c22e98b38(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T03:40:24,623 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 53a70bd4899ab7aa137c5ffbd98f8de7: 2024-11-22T03:40:24,623 INFO [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., storeName=53a70bd4899ab7aa137c5ffbd98f8de7/A, priority=13, startTime=1732246824518; duration=0sec 2024-11-22T03:40:24,623 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7. as already flushing 2024-11-22T03:40:24,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(8855): Flush requested on 53a70bd4899ab7aa137c5ffbd98f8de7 2024-11-22T03:40:24,623 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T03:40:24,623 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 53a70bd4899ab7aa137c5ffbd98f8de7:A 2024-11-22T03:40:24,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411226554a47af350405e9eca14c72b988e3e_53a70bd4899ab7aa137c5ffbd98f8de7 is 50, key is test_row_0/A:col10/1732246823983/Put/seqid=0 2024-11-22T03:40:24,636 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073741972_1148 (size=12104) 2024-11-22T03:40:24,641 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:24,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34746 deadline: 1732246884629, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:24,641 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:24,641 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:24,641 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:24,642 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:40:24,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34684 deadline: 1732246884629, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:24,642 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:24,642 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:24,642 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:24,642 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:40:24,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34754 deadline: 1732246884632, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:24,642 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:24,643 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:24,643 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:24,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34770 deadline: 1732246884635, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:24,643 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:40:24,643 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:24,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34728 deadline: 1732246884636, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:24,643 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:24,644 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:24,644 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:40:24,645 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:24,645 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:24,645 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:40:24,650 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/.tmp/C/f935efd1741a4353b24f1b2e16390aa2 as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/C/f935efd1741a4353b24f1b2e16390aa2 2024-11-22T03:40:24,664 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073741973_1149 (size=12154) 2024-11-22T03:40:24,665 INFO [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 53a70bd4899ab7aa137c5ffbd98f8de7/C of 53a70bd4899ab7aa137c5ffbd98f8de7 into f935efd1741a4353b24f1b2e16390aa2(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T03:40:24,665 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:24,665 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 53a70bd4899ab7aa137c5ffbd98f8de7: 2024-11-22T03:40:24,665 INFO [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., storeName=53a70bd4899ab7aa137c5ffbd98f8de7/C, priority=13, startTime=1732246824518; duration=0sec 2024-11-22T03:40:24,665 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T03:40:24,665 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 53a70bd4899ab7aa137c5ffbd98f8de7:C 2024-11-22T03:40:24,671 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411226554a47af350405e9eca14c72b988e3e_53a70bd4899ab7aa137c5ffbd98f8de7 to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411226554a47af350405e9eca14c72b988e3e_53a70bd4899ab7aa137c5ffbd98f8de7 2024-11-22T03:40:24,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/.tmp/A/c250eb0bdb3442a6a4d45a5802f450d5, store: [table=TestAcidGuarantees family=A region=53a70bd4899ab7aa137c5ffbd98f8de7] 2024-11-22T03:40:24,673 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/.tmp/A/c250eb0bdb3442a6a4d45a5802f450d5 is 175, key is test_row_0/A:col10/1732246823983/Put/seqid=0 2024-11-22T03:40:24,684 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073741974_1150 (size=30955) 2024-11-22T03:40:24,749 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:24,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34728 deadline: 1732246884749, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:24,750 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:24,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34684 deadline: 1732246884749, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:24,750 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:24,750 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:24,750 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:24,750 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:40:24,750 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:24,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34770 deadline: 1732246884749, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:24,750 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:24,750 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:40:24,751 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:24,751 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:24,751 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:24,751 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:40:24,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34754 deadline: 1732246884749, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:24,751 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:24,751 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:24,751 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:40:24,962 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:24,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34728 deadline: 1732246884958, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:24,962 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:24,962 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:24,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34684 deadline: 1732246884958, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:24,963 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:24,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34770 deadline: 1732246884959, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:24,963 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:24,963 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:24,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34754 deadline: 1732246884959, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:24,963 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:40:24,963 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:24,963 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:24,963 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:40:24,963 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:24,963 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:24,963 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:40:24,964 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:24,964 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:24,964 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:40:25,087 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=78, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/.tmp/A/c250eb0bdb3442a6a4d45a5802f450d5 2024-11-22T03:40:25,106 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/.tmp/B/01854badc16c4a9191303248880dda95 is 50, key is test_row_0/B:col10/1732246823983/Put/seqid=0 2024-11-22T03:40:25,115 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073741975_1151 (size=12001) 2024-11-22T03:40:25,149 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:25,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34746 deadline: 1732246885148, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:25,151 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:25,151 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:25,151 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:40:25,272 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:25,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34728 deadline: 1732246885268, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:25,273 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:25,273 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:25,273 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:25,273 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:40:25,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34684 deadline: 1732246885269, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:25,274 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:25,274 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:25,274 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:25,274 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:40:25,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34770 deadline: 1732246885269, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:25,274 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:25,274 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:25,274 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:25,274 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:40:25,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34754 deadline: 1732246885269, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:25,275 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:25,275 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:25,275 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:40:25,516 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=78 (bloomFilter=true), to=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/.tmp/B/01854badc16c4a9191303248880dda95 2024-11-22T03:40:25,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/.tmp/C/ae9be35fafae4f56ab4c55048983720a is 50, key is test_row_0/C:col10/1732246823983/Put/seqid=0 2024-11-22T03:40:25,539 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073741976_1152 (size=12001) 2024-11-22T03:40:25,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-22T03:40:25,780 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:25,780 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:25,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34728 deadline: 1732246885779, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:25,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34770 deadline: 1732246885779, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:25,780 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:25,780 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:25,780 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:25,780 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:25,781 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:40:25,781 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:40:25,780 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:25,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34754 deadline: 1732246885779, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:25,781 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:25,781 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:25,781 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:40:25,789 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:25,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34684 deadline: 1732246885789, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:25,790 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:25,790 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:25,790 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:40:25,941 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=78 (bloomFilter=true), to=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/.tmp/C/ae9be35fafae4f56ab4c55048983720a 2024-11-22T03:40:25,949 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/.tmp/A/c250eb0bdb3442a6a4d45a5802f450d5 as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/A/c250eb0bdb3442a6a4d45a5802f450d5 2024-11-22T03:40:25,954 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/A/c250eb0bdb3442a6a4d45a5802f450d5, entries=150, sequenceid=78, filesize=30.2 K 2024-11-22T03:40:25,955 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/.tmp/B/01854badc16c4a9191303248880dda95 as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/B/01854badc16c4a9191303248880dda95 2024-11-22T03:40:25,960 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/B/01854badc16c4a9191303248880dda95, entries=150, sequenceid=78, filesize=11.7 K 2024-11-22T03:40:25,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/.tmp/C/ae9be35fafae4f56ab4c55048983720a as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/C/ae9be35fafae4f56ab4c55048983720a 2024-11-22T03:40:25,970 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/C/ae9be35fafae4f56ab4c55048983720a, entries=150, sequenceid=78, filesize=11.7 K 2024-11-22T03:40:25,973 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 53a70bd4899ab7aa137c5ffbd98f8de7 in 1364ms, sequenceid=78, compaction requested=false 2024-11-22T03:40:25,973 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2603): Flush status journal for 53a70bd4899ab7aa137c5ffbd98f8de7: 2024-11-22T03:40:25,973 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7. 2024-11-22T03:40:25,973 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=43 2024-11-22T03:40:25,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42627 {}] master.HMaster(4169): Remote procedure done, pid=43 2024-11-22T03:40:25,976 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=43, resume processing ppid=42 2024-11-22T03:40:25,976 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=43, ppid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 4.4680 sec 2024-11-22T03:40:25,978 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees in 4.4810 sec 2024-11-22T03:40:26,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(8855): Flush requested on 53a70bd4899ab7aa137c5ffbd98f8de7 2024-11-22T03:40:26,171 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-22T03:40:26,172 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 53a70bd4899ab7aa137c5ffbd98f8de7 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-22T03:40:26,201 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112289917da1df434f2ab99e6a32b9b95d6a_53a70bd4899ab7aa137c5ffbd98f8de7 is 50, key is test_row_0/A:col10/1732246824629/Put/seqid=0 2024-11-22T03:40:26,213 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073741977_1153 (size=14594) 2024-11-22T03:40:26,217 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:26,222 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112289917da1df434f2ab99e6a32b9b95d6a_53a70bd4899ab7aa137c5ffbd98f8de7 to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112289917da1df434f2ab99e6a32b9b95d6a_53a70bd4899ab7aa137c5ffbd98f8de7 2024-11-22T03:40:26,225 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/.tmp/A/6caebab891ba492e96765f14a60ef5ec, store: [table=TestAcidGuarantees family=A region=53a70bd4899ab7aa137c5ffbd98f8de7] 2024-11-22T03:40:26,225 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/.tmp/A/6caebab891ba492e96765f14a60ef5ec is 175, key is test_row_0/A:col10/1732246824629/Put/seqid=0 2024-11-22T03:40:26,239 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073741978_1154 (size=39549) 2024-11-22T03:40:26,280 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:26,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34746 deadline: 1732246886277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:26,281 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:26,281 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:26,281 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:40:26,389 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:26,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34746 deadline: 1732246886389, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:26,390 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:26,390 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:26,390 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:40:26,600 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:26,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34746 deadline: 1732246886598, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:26,600 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:26,600 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:26,600 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:40:26,639 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=92, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/.tmp/A/6caebab891ba492e96765f14a60ef5ec 2024-11-22T03:40:26,646 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/.tmp/B/0d78c199960542dcb3920cc0be2e9e3a is 50, key is test_row_0/B:col10/1732246824629/Put/seqid=0 2024-11-22T03:40:26,650 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073741979_1155 (size=12001) 2024-11-22T03:40:26,790 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:26,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34770 deadline: 1732246886789, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:26,791 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:26,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34754 deadline: 1732246886789, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:26,795 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-22T03:40:26,795 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to c85114ed5096,42545,1732246786482 for 'test_row_1' in 53a70bd4899ab7aa137c5ffbd98f8de7 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2157 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-22T03:40:26,796 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:26,796 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:26,796 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:40:26,797 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to c85114ed5096,42545,1732246786482 for 'test_row_2' in 53a70bd4899ab7aa137c5ffbd98f8de7 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2160 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-22T03:40:26,798 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:26,798 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:26,798 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:40:26,807 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:26,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34728 deadline: 1732246886804, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:26,808 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to c85114ed5096,42545,1732246786482 for 'test_row_0' in 53a70bd4899ab7aa137c5ffbd98f8de7 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2171 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-22T03:40:26,808 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:26,808 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:26,809 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:40:26,809 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:26,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34684 deadline: 1732246886809, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:26,809 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to c85114ed5096,42545,1732246786482 for 'test_row_0' in 53a70bd4899ab7aa137c5ffbd98f8de7 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2180 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-22T03:40:26,810 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:26,810 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:26,810 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:40:26,913 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:26,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34746 deadline: 1732246886909, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:26,913 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:26,913 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:26,913 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:40:27,050 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=92 (bloomFilter=true), to=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/.tmp/B/0d78c199960542dcb3920cc0be2e9e3a 2024-11-22T03:40:27,059 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/.tmp/C/ea6858231cd142fb95b139d52dac81e1 is 50, key is test_row_0/C:col10/1732246824629/Put/seqid=0 2024-11-22T03:40:27,063 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073741980_1156 (size=12001) 2024-11-22T03:40:27,431 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:27,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34746 deadline: 1732246887429, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:27,432 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:27,432 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:27,432 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:40:27,464 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=92 (bloomFilter=true), to=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/.tmp/C/ea6858231cd142fb95b139d52dac81e1 2024-11-22T03:40:27,469 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/.tmp/A/6caebab891ba492e96765f14a60ef5ec as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/A/6caebab891ba492e96765f14a60ef5ec 2024-11-22T03:40:27,473 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/A/6caebab891ba492e96765f14a60ef5ec, entries=200, sequenceid=92, filesize=38.6 K 2024-11-22T03:40:27,474 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/.tmp/B/0d78c199960542dcb3920cc0be2e9e3a as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/B/0d78c199960542dcb3920cc0be2e9e3a 2024-11-22T03:40:27,478 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/B/0d78c199960542dcb3920cc0be2e9e3a, entries=150, sequenceid=92, filesize=11.7 K 2024-11-22T03:40:27,481 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/.tmp/C/ea6858231cd142fb95b139d52dac81e1 as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/C/ea6858231cd142fb95b139d52dac81e1 2024-11-22T03:40:27,489 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/C/ea6858231cd142fb95b139d52dac81e1, entries=150, sequenceid=92, filesize=11.7 K 2024-11-22T03:40:27,493 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 53a70bd4899ab7aa137c5ffbd98f8de7 in 1322ms, sequenceid=92, compaction requested=true 2024-11-22T03:40:27,493 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 53a70bd4899ab7aa137c5ffbd98f8de7: 2024-11-22T03:40:27,493 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 53a70bd4899ab7aa137c5ffbd98f8de7:A, priority=-2147483648, current under compaction store size is 1 2024-11-22T03:40:27,493 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T03:40:27,493 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T03:40:27,493 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T03:40:27,493 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 53a70bd4899ab7aa137c5ffbd98f8de7:B, priority=-2147483648, current under compaction store size is 2 2024-11-22T03:40:27,493 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T03:40:27,493 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 53a70bd4899ab7aa137c5ffbd98f8de7:C, priority=-2147483648, current under compaction store size is 3 2024-11-22T03:40:27,494 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T03:40:27,495 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T03:40:27,495 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101562 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T03:40:27,495 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HStore(1541): 53a70bd4899ab7aa137c5ffbd98f8de7/B is initiating minor compaction (all files) 2024-11-22T03:40:27,495 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.HStore(1541): 53a70bd4899ab7aa137c5ffbd98f8de7/A is initiating minor compaction (all files) 2024-11-22T03:40:27,495 INFO [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 53a70bd4899ab7aa137c5ffbd98f8de7/B in TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7. 2024-11-22T03:40:27,495 INFO [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 53a70bd4899ab7aa137c5ffbd98f8de7/A in TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7. 2024-11-22T03:40:27,495 INFO [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/B/0fe94b926f4b4fa2a0d02cc23e244525, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/B/01854badc16c4a9191303248880dda95, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/B/0d78c199960542dcb3920cc0be2e9e3a] into tmpdir=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/.tmp, totalSize=35.3 K 2024-11-22T03:40:27,495 INFO [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/A/11fcb99717c4471d9f90041c22e98b38, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/A/c250eb0bdb3442a6a4d45a5802f450d5, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/A/6caebab891ba492e96765f14a60ef5ec] into tmpdir=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/.tmp, totalSize=99.2 K 2024-11-22T03:40:27,495 INFO [RS:0;c85114ed5096:42545-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7. 2024-11-22T03:40:27,495 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7. files: [hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/A/11fcb99717c4471d9f90041c22e98b38, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/A/c250eb0bdb3442a6a4d45a5802f450d5, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/A/6caebab891ba492e96765f14a60ef5ec] 2024-11-22T03:40:27,496 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] compactions.Compactor(225): Compacting 0fe94b926f4b4fa2a0d02cc23e244525, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1732246822763 2024-11-22T03:40:27,496 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] compactions.Compactor(225): Compacting 11fcb99717c4471d9f90041c22e98b38, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1732246822763 2024-11-22T03:40:27,496 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] compactions.Compactor(225): Compacting c250eb0bdb3442a6a4d45a5802f450d5, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=78, earliestPutTs=1732246823983 2024-11-22T03:40:27,496 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] compactions.Compactor(225): Compacting 01854badc16c4a9191303248880dda95, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=78, earliestPutTs=1732246823983 2024-11-22T03:40:27,496 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6caebab891ba492e96765f14a60ef5ec, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1732246824629 2024-11-22T03:40:27,497 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] compactions.Compactor(225): Compacting 0d78c199960542dcb3920cc0be2e9e3a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1732246824629 2024-11-22T03:40:27,506 INFO [RS:0;c85114ed5096:42545-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 53a70bd4899ab7aa137c5ffbd98f8de7#B#compaction#135 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T03:40:27,507 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/.tmp/B/5d02aff14b9c4ee3bab729c0f3a53633 is 50, key is test_row_0/B:col10/1732246824629/Put/seqid=0 2024-11-22T03:40:27,508 INFO [RS:0;c85114ed5096:42545-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=53a70bd4899ab7aa137c5ffbd98f8de7] 2024-11-22T03:40:27,514 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073741981_1157 (size=12207) 2024-11-22T03:40:27,517 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024112205b2a1d45fe444fc94e1c44d92df3b04_53a70bd4899ab7aa137c5ffbd98f8de7 store=[table=TestAcidGuarantees family=A region=53a70bd4899ab7aa137c5ffbd98f8de7] 2024-11-22T03:40:27,521 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024112205b2a1d45fe444fc94e1c44d92df3b04_53a70bd4899ab7aa137c5ffbd98f8de7, store=[table=TestAcidGuarantees family=A region=53a70bd4899ab7aa137c5ffbd98f8de7] 2024-11-22T03:40:27,522 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112205b2a1d45fe444fc94e1c44d92df3b04_53a70bd4899ab7aa137c5ffbd98f8de7 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=53a70bd4899ab7aa137c5ffbd98f8de7] 2024-11-22T03:40:27,534 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073741982_1158 (size=4469) 2024-11-22T03:40:27,923 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/.tmp/B/5d02aff14b9c4ee3bab729c0f3a53633 as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/B/5d02aff14b9c4ee3bab729c0f3a53633 2024-11-22T03:40:27,929 INFO [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 53a70bd4899ab7aa137c5ffbd98f8de7/B of 53a70bd4899ab7aa137c5ffbd98f8de7 into 5d02aff14b9c4ee3bab729c0f3a53633(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T03:40:27,929 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 53a70bd4899ab7aa137c5ffbd98f8de7: 2024-11-22T03:40:27,929 INFO [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., storeName=53a70bd4899ab7aa137c5ffbd98f8de7/B, priority=13, startTime=1732246827493; duration=0sec 2024-11-22T03:40:27,930 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T03:40:27,930 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 53a70bd4899ab7aa137c5ffbd98f8de7:B 2024-11-22T03:40:27,930 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T03:40:27,946 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T03:40:27,947 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HStore(1541): 53a70bd4899ab7aa137c5ffbd98f8de7/C is initiating minor compaction (all files) 2024-11-22T03:40:27,947 INFO [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 53a70bd4899ab7aa137c5ffbd98f8de7/C in TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7. 2024-11-22T03:40:27,947 INFO [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/C/f935efd1741a4353b24f1b2e16390aa2, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/C/ae9be35fafae4f56ab4c55048983720a, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/C/ea6858231cd142fb95b139d52dac81e1] into tmpdir=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/.tmp, totalSize=35.3 K 2024-11-22T03:40:27,948 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] compactions.Compactor(225): Compacting f935efd1741a4353b24f1b2e16390aa2, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1732246822763 2024-11-22T03:40:27,949 INFO [RS:0;c85114ed5096:42545-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 53a70bd4899ab7aa137c5ffbd98f8de7#A#compaction#136 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T03:40:27,950 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/.tmp/A/37890ab5a23441c98c79cd41491f6b91 is 175, key is test_row_0/A:col10/1732246824629/Put/seqid=0 2024-11-22T03:40:27,950 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] compactions.Compactor(225): Compacting ae9be35fafae4f56ab4c55048983720a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=78, earliestPutTs=1732246823983 2024-11-22T03:40:27,952 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] compactions.Compactor(225): Compacting ea6858231cd142fb95b139d52dac81e1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1732246824629 2024-11-22T03:40:27,962 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073741983_1159 (size=31161) 2024-11-22T03:40:27,964 INFO [RS:0;c85114ed5096:42545-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 53a70bd4899ab7aa137c5ffbd98f8de7#C#compaction#137 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T03:40:27,965 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/.tmp/C/88d255ccef494fa28358c67603862c6b is 50, key is test_row_0/C:col10/1732246824629/Put/seqid=0 2024-11-22T03:40:27,971 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/.tmp/A/37890ab5a23441c98c79cd41491f6b91 as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/A/37890ab5a23441c98c79cd41491f6b91 2024-11-22T03:40:27,972 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073741984_1160 (size=12207) 2024-11-22T03:40:27,977 INFO [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 53a70bd4899ab7aa137c5ffbd98f8de7/A of 53a70bd4899ab7aa137c5ffbd98f8de7 into 37890ab5a23441c98c79cd41491f6b91(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T03:40:27,977 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 53a70bd4899ab7aa137c5ffbd98f8de7: 2024-11-22T03:40:27,977 INFO [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., storeName=53a70bd4899ab7aa137c5ffbd98f8de7/A, priority=13, startTime=1732246827493; duration=0sec 2024-11-22T03:40:27,978 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T03:40:27,978 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 53a70bd4899ab7aa137c5ffbd98f8de7:A 2024-11-22T03:40:27,979 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/.tmp/C/88d255ccef494fa28358c67603862c6b as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/C/88d255ccef494fa28358c67603862c6b 2024-11-22T03:40:27,986 INFO [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 53a70bd4899ab7aa137c5ffbd98f8de7/C of 53a70bd4899ab7aa137c5ffbd98f8de7 into 88d255ccef494fa28358c67603862c6b(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T03:40:27,986 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 53a70bd4899ab7aa137c5ffbd98f8de7: 2024-11-22T03:40:27,986 INFO [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., storeName=53a70bd4899ab7aa137c5ffbd98f8de7/C, priority=13, startTime=1732246827493; duration=0sec 2024-11-22T03:40:27,986 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T03:40:27,986 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 53a70bd4899ab7aa137c5ffbd98f8de7:C 2024-11-22T03:40:28,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(8855): Flush requested on 53a70bd4899ab7aa137c5ffbd98f8de7 2024-11-22T03:40:28,451 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-22T03:40:28,451 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 53a70bd4899ab7aa137c5ffbd98f8de7 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-22T03:40:28,459 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241122f72aae3cca274599b4c692314ac4b893_53a70bd4899ab7aa137c5ffbd98f8de7 is 50, key is test_row_0/A:col10/1732246826270/Put/seqid=0 2024-11-22T03:40:28,464 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073741985_1161 (size=14594) 2024-11-22T03:40:28,498 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:28,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34746 deadline: 1732246888496, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:28,499 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:28,499 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:28,499 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:40:28,612 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:28,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34746 deadline: 1732246888609, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:28,613 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:28,613 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:28,613 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:40:28,821 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:28,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34770 deadline: 1732246888819, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:28,822 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:28,822 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:28,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34754 deadline: 1732246888819, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:28,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34684 deadline: 1732246888819, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:28,822 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:28,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34746 deadline: 1732246888819, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:28,822 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to c85114ed5096,42545,1732246786482 for 'test_row_1' in 53a70bd4899ab7aa137c5ffbd98f8de7 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4187 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-22T03:40:28,823 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:28,823 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:28,822 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to c85114ed5096,42545,1732246786482 for 'test_row_2' in 53a70bd4899ab7aa137c5ffbd98f8de7 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4190 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-22T03:40:28,822 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to c85114ed5096,42545,1732246786482 for 'test_row_0' in 53a70bd4899ab7aa137c5ffbd98f8de7 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4193 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-22T03:40:28,823 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:28,823 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:28,823 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:28,823 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:28,823 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:28,823 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:40:28,823 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:40:28,823 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:28,823 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:40:28,823 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:40:28,831 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:28,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34728 deadline: 1732246888828, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:28,831 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to c85114ed5096,42545,1732246786482 for 'test_row_0' in 53a70bd4899ab7aa137c5ffbd98f8de7 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4195 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-22T03:40:28,832 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:28,832 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:28,832 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:40:28,864 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:28,868 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241122f72aae3cca274599b4c692314ac4b893_53a70bd4899ab7aa137c5ffbd98f8de7 to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241122f72aae3cca274599b4c692314ac4b893_53a70bd4899ab7aa137c5ffbd98f8de7 2024-11-22T03:40:28,869 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/.tmp/A/3962f084c74946c1a72c72b791c92b7a, store: [table=TestAcidGuarantees family=A region=53a70bd4899ab7aa137c5ffbd98f8de7] 2024-11-22T03:40:28,869 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/.tmp/A/3962f084c74946c1a72c72b791c92b7a is 175, key is test_row_0/A:col10/1732246826270/Put/seqid=0 2024-11-22T03:40:28,873 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073741986_1162 (size=39549) 2024-11-22T03:40:29,129 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:29,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34746 deadline: 1732246889128, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:29,130 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:29,130 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:29,130 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:40:29,273 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=120, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/.tmp/A/3962f084c74946c1a72c72b791c92b7a 2024-11-22T03:40:29,281 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/.tmp/B/566cd0fe6c4b4a17b8eb691d57317312 is 50, key is test_row_0/B:col10/1732246826270/Put/seqid=0 2024-11-22T03:40:29,286 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073741987_1163 (size=12001) 2024-11-22T03:40:29,639 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:29,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34746 deadline: 1732246889638, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:29,640 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:29,640 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:29,640 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:40:29,687 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=120 (bloomFilter=true), to=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/.tmp/B/566cd0fe6c4b4a17b8eb691d57317312 2024-11-22T03:40:29,703 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/.tmp/C/13ff308f4c924ed097f50cedcf9e4ccb is 50, key is test_row_0/C:col10/1732246826270/Put/seqid=0 2024-11-22T03:40:29,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-22T03:40:29,718 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073741988_1164 (size=12001) 2024-11-22T03:40:29,719 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-22T03:40:29,719 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42627 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-22T03:40:29,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42627 {}] procedure2.ProcedureExecutor(1139): Stored pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees 2024-11-22T03:40:29,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-22T03:40:29,721 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-22T03:40:29,722 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-22T03:40:29,722 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=45, ppid=44, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-22T03:40:29,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-22T03:40:29,874 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42545 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-22T03:40:29,874 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7. 2024-11-22T03:40:29,874 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7. as already flushing 2024-11-22T03:40:29,874 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7. 2024-11-22T03:40:29,875 ERROR [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 53a70bd4899ab7aa137c5ffbd98f8de7, NAME => 'TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:40:29,875 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 53a70bd4899ab7aa137c5ffbd98f8de7, NAME => 'TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:40:29,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42627 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 53a70bd4899ab7aa137c5ffbd98f8de7, NAME => 'TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 53a70bd4899ab7aa137c5ffbd98f8de7, NAME => 'TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:40:30,027 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42545 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-22T03:40:30,027 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7. 2024-11-22T03:40:30,027 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7. as already flushing 2024-11-22T03:40:30,027 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7. 2024-11-22T03:40:30,027 ERROR [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 53a70bd4899ab7aa137c5ffbd98f8de7, NAME => 'TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:40:30,027 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 53a70bd4899ab7aa137c5ffbd98f8de7, NAME => 'TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:40:30,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42627 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 53a70bd4899ab7aa137c5ffbd98f8de7, NAME => 'TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 53a70bd4899ab7aa137c5ffbd98f8de7, NAME => 'TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:40:30,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-22T03:40:30,121 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=120 (bloomFilter=true), to=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/.tmp/C/13ff308f4c924ed097f50cedcf9e4ccb 2024-11-22T03:40:30,126 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/.tmp/A/3962f084c74946c1a72c72b791c92b7a as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/A/3962f084c74946c1a72c72b791c92b7a 2024-11-22T03:40:30,130 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/A/3962f084c74946c1a72c72b791c92b7a, entries=200, sequenceid=120, filesize=38.6 K 2024-11-22T03:40:30,131 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/.tmp/B/566cd0fe6c4b4a17b8eb691d57317312 as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/B/566cd0fe6c4b4a17b8eb691d57317312 2024-11-22T03:40:30,139 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/B/566cd0fe6c4b4a17b8eb691d57317312, entries=150, sequenceid=120, filesize=11.7 K 2024-11-22T03:40:30,140 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/.tmp/C/13ff308f4c924ed097f50cedcf9e4ccb as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/C/13ff308f4c924ed097f50cedcf9e4ccb 2024-11-22T03:40:30,144 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/C/13ff308f4c924ed097f50cedcf9e4ccb, entries=150, sequenceid=120, filesize=11.7 K 2024-11-22T03:40:30,145 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 53a70bd4899ab7aa137c5ffbd98f8de7 in 1694ms, sequenceid=120, compaction requested=false 2024-11-22T03:40:30,145 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 53a70bd4899ab7aa137c5ffbd98f8de7: 2024-11-22T03:40:30,180 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42545 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-22T03:40:30,180 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7. 2024-11-22T03:40:30,180 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2902): Flushing 53a70bd4899ab7aa137c5ffbd98f8de7 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-22T03:40:30,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112263131f685b454a0da2e7ae7daa54ab61_53a70bd4899ab7aa137c5ffbd98f8de7 is 50, key is test_row_0/A:col10/1732246828494/Put/seqid=0 2024-11-22T03:40:30,196 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073741989_1165 (size=12204) 2024-11-22T03:40:30,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-22T03:40:30,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:30,618 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112263131f685b454a0da2e7ae7daa54ab61_53a70bd4899ab7aa137c5ffbd98f8de7 to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112263131f685b454a0da2e7ae7daa54ab61_53a70bd4899ab7aa137c5ffbd98f8de7 2024-11-22T03:40:30,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/.tmp/A/5e32ef50efbc490fa570b73d9c23882d, store: [table=TestAcidGuarantees family=A region=53a70bd4899ab7aa137c5ffbd98f8de7] 2024-11-22T03:40:30,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/.tmp/A/5e32ef50efbc490fa570b73d9c23882d is 175, key is test_row_0/A:col10/1732246828494/Put/seqid=0 2024-11-22T03:40:30,623 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073741990_1166 (size=31005) 2024-11-22T03:40:30,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(8855): Flush requested on 53a70bd4899ab7aa137c5ffbd98f8de7 2024-11-22T03:40:30,650 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7. as already flushing 2024-11-22T03:40:30,727 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:30,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34746 deadline: 1732246890727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:30,728 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:30,728 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:30,728 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:40:30,843 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:30,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34746 deadline: 1732246890839, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:30,843 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:30,843 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:30,843 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:40:30,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-22T03:40:31,024 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=131, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/.tmp/A/5e32ef50efbc490fa570b73d9c23882d 2024-11-22T03:40:31,035 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/.tmp/B/0eb207b7aae442e8b69682b733b11ef8 is 50, key is test_row_0/B:col10/1732246828494/Put/seqid=0 2024-11-22T03:40:31,040 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073741991_1167 (size=12051) 2024-11-22T03:40:31,049 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:31,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34746 deadline: 1732246891049, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:31,050 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:31,050 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:31,050 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:40:31,359 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:31,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34746 deadline: 1732246891358, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:31,359 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:31,359 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:31,359 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:40:31,440 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=131 (bloomFilter=true), to=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/.tmp/B/0eb207b7aae442e8b69682b733b11ef8 2024-11-22T03:40:31,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/.tmp/C/5a654c2fcc654d899dbc108782885a6e is 50, key is test_row_0/C:col10/1732246828494/Put/seqid=0 2024-11-22T03:40:31,458 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073741992_1168 (size=12051) 2024-11-22T03:40:31,674 INFO [master/c85114ed5096:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-11-22T03:40:31,674 INFO [master/c85114ed5096:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-11-22T03:40:31,859 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=131 (bloomFilter=true), to=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/.tmp/C/5a654c2fcc654d899dbc108782885a6e 2024-11-22T03:40:31,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-22T03:40:31,869 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/.tmp/A/5e32ef50efbc490fa570b73d9c23882d as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/A/5e32ef50efbc490fa570b73d9c23882d 2024-11-22T03:40:31,873 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:31,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34746 deadline: 1732246891868, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:31,873 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/A/5e32ef50efbc490fa570b73d9c23882d, entries=150, sequenceid=131, filesize=30.3 K 2024-11-22T03:40:31,873 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:31,873 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:31,874 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:40:31,875 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/.tmp/B/0eb207b7aae442e8b69682b733b11ef8 as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/B/0eb207b7aae442e8b69682b733b11ef8 2024-11-22T03:40:31,879 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/B/0eb207b7aae442e8b69682b733b11ef8, entries=150, sequenceid=131, filesize=11.8 K 2024-11-22T03:40:31,880 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/.tmp/C/5a654c2fcc654d899dbc108782885a6e as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/C/5a654c2fcc654d899dbc108782885a6e 2024-11-22T03:40:31,885 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/C/5a654c2fcc654d899dbc108782885a6e, entries=150, sequenceid=131, filesize=11.8 K 2024-11-22T03:40:31,886 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 53a70bd4899ab7aa137c5ffbd98f8de7 in 1706ms, sequenceid=131, compaction requested=true 2024-11-22T03:40:31,886 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2603): Flush status journal for 53a70bd4899ab7aa137c5ffbd98f8de7: 2024-11-22T03:40:31,886 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7. 2024-11-22T03:40:31,886 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=45 2024-11-22T03:40:31,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42627 {}] master.HMaster(4169): Remote procedure done, pid=45 2024-11-22T03:40:31,889 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=45, resume processing ppid=44 2024-11-22T03:40:31,889 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=45, ppid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1650 sec 2024-11-22T03:40:31,892 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees in 2.1710 sec 2024-11-22T03:40:32,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(8855): Flush requested on 53a70bd4899ab7aa137c5ffbd98f8de7 2024-11-22T03:40:32,842 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-22T03:40:32,842 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 53a70bd4899ab7aa137c5ffbd98f8de7 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-22T03:40:32,850 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241122eee8581ccae84b03b9dd79de682c167d_53a70bd4899ab7aa137c5ffbd98f8de7 is 50, key is test_row_0/A:col10/1732246832841/Put/seqid=0 2024-11-22T03:40:32,854 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073741993_1169 (size=14794) 2024-11-22T03:40:32,865 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:32,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34684 deadline: 1732246892862, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:32,865 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:32,865 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:32,865 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:32,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34754 deadline: 1732246892864, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:32,865 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:40:32,866 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:32,866 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:32,866 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:40:32,867 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:32,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34770 deadline: 1732246892865, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:32,867 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:32,867 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:32,867 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:40:32,870 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:32,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34728 deadline: 1732246892868, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:32,870 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to c85114ed5096,42545,1732246786482 for 'test_row_0' in 53a70bd4899ab7aa137c5ffbd98f8de7 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8234 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-22T03:40:32,871 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:32,871 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:32,871 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:40:32,889 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:32,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34746 deadline: 1732246892888, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:32,890 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to c85114ed5096,42545,1732246786482 for 'test_row_0' in 53a70bd4899ab7aa137c5ffbd98f8de7 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2163 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-22T03:40:32,890 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:32,890 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:32,890 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:40:32,968 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:32,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34684 deadline: 1732246892968, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:32,969 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:32,969 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:32,969 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:40:32,969 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:32,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34754 deadline: 1732246892969, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:32,970 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:32,970 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:32,970 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:40:32,979 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:32,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34770 deadline: 1732246892978, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:32,979 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:32,979 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:32,979 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:40:33,181 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:33,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34684 deadline: 1732246893178, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:33,181 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:33,181 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:33,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34754 deadline: 1732246893178, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:33,181 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:33,181 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:40:33,181 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:33,181 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:33,182 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:40:33,191 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:33,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34770 deadline: 1732246893188, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:33,191 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:33,191 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:33,191 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:40:33,255 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:33,259 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241122eee8581ccae84b03b9dd79de682c167d_53a70bd4899ab7aa137c5ffbd98f8de7 to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241122eee8581ccae84b03b9dd79de682c167d_53a70bd4899ab7aa137c5ffbd98f8de7 2024-11-22T03:40:33,260 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/.tmp/A/c17d0df2f5a94a859b36aecabc975e26, store: [table=TestAcidGuarantees family=A region=53a70bd4899ab7aa137c5ffbd98f8de7] 2024-11-22T03:40:33,261 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/.tmp/A/c17d0df2f5a94a859b36aecabc975e26 is 175, key is test_row_0/A:col10/1732246832841/Put/seqid=0 2024-11-22T03:40:33,264 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073741994_1170 (size=39749) 2024-11-22T03:40:33,491 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:33,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34684 deadline: 1732246893489, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:33,492 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:33,492 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:33,492 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:33,492 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:40:33,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34754 deadline: 1732246893489, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:33,492 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:33,492 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:33,492 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:40:33,498 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:33,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34770 deadline: 1732246893498, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:33,499 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:33,499 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:33,499 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:40:33,665 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=157, memsize=51.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/.tmp/A/c17d0df2f5a94a859b36aecabc975e26 2024-11-22T03:40:33,673 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/.tmp/B/78d718ba93ee40c5a5bbd69e83675d39 is 50, key is test_row_0/B:col10/1732246832841/Put/seqid=0 2024-11-22T03:40:33,677 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073741995_1171 (size=12151) 2024-11-22T03:40:33,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-22T03:40:33,889 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-22T03:40:33,889 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42627 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-22T03:40:33,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42627 {}] procedure2.ProcedureExecutor(1139): Stored pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees 2024-11-22T03:40:33,891 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-22T03:40:33,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-22T03:40:33,892 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-22T03:40:33,892 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=47, ppid=46, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-22T03:40:33,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-22T03:40:34,002 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:34,002 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:34,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34754 deadline: 1732246893998, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:34,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34684 deadline: 1732246893998, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:34,003 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:34,003 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:34,003 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:34,003 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:34,003 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:40:34,003 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:40:34,012 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:34,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34770 deadline: 1732246894009, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:34,012 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:34,012 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:34,012 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:40:34,045 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42545 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-22T03:40:34,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7. 2024-11-22T03:40:34,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7. as already flushing 2024-11-22T03:40:34,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7. 2024-11-22T03:40:34,045 ERROR [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 53a70bd4899ab7aa137c5ffbd98f8de7, NAME => 'TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:40:34,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 53a70bd4899ab7aa137c5ffbd98f8de7, NAME => 'TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:40:34,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42627 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 53a70bd4899ab7aa137c5ffbd98f8de7, NAME => 'TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 53a70bd4899ab7aa137c5ffbd98f8de7, NAME => 'TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:40:34,078 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=157 (bloomFilter=true), to=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/.tmp/B/78d718ba93ee40c5a5bbd69e83675d39 2024-11-22T03:40:34,086 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/.tmp/C/602d166562cc4643a27f5cb843e33ddd is 50, key is test_row_0/C:col10/1732246832841/Put/seqid=0 2024-11-22T03:40:34,089 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073741996_1172 (size=12151) 2024-11-22T03:40:34,198 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42545 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-22T03:40:34,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7. 2024-11-22T03:40:34,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7. as already flushing 2024-11-22T03:40:34,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7. 2024-11-22T03:40:34,198 ERROR [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 53a70bd4899ab7aa137c5ffbd98f8de7, NAME => 'TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:40:34,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 53a70bd4899ab7aa137c5ffbd98f8de7, NAME => 'TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:40:34,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42627 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 53a70bd4899ab7aa137c5ffbd98f8de7, NAME => 'TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 53a70bd4899ab7aa137c5ffbd98f8de7, NAME => 'TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:40:34,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-22T03:40:34,351 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42545 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-22T03:40:34,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7. 2024-11-22T03:40:34,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7. as already flushing 2024-11-22T03:40:34,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7. 2024-11-22T03:40:34,352 ERROR [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 53a70bd4899ab7aa137c5ffbd98f8de7, NAME => 'TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:40:34,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 53a70bd4899ab7aa137c5ffbd98f8de7, NAME => 'TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:40:34,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42627 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 53a70bd4899ab7aa137c5ffbd98f8de7, NAME => 'TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 53a70bd4899ab7aa137c5ffbd98f8de7, NAME => 'TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:40:34,491 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=157 (bloomFilter=true), to=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/.tmp/C/602d166562cc4643a27f5cb843e33ddd 2024-11-22T03:40:34,498 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/.tmp/A/c17d0df2f5a94a859b36aecabc975e26 as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/A/c17d0df2f5a94a859b36aecabc975e26 2024-11-22T03:40:34,502 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/A/c17d0df2f5a94a859b36aecabc975e26, entries=200, sequenceid=157, filesize=38.8 K 2024-11-22T03:40:34,503 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/.tmp/B/78d718ba93ee40c5a5bbd69e83675d39 as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/B/78d718ba93ee40c5a5bbd69e83675d39 2024-11-22T03:40:34,505 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42545 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-22T03:40:34,506 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7. 2024-11-22T03:40:34,506 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7. as already flushing 2024-11-22T03:40:34,506 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7. 2024-11-22T03:40:34,506 ERROR [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 53a70bd4899ab7aa137c5ffbd98f8de7, NAME => 'TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:40:34,506 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 53a70bd4899ab7aa137c5ffbd98f8de7, NAME => 'TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:40:34,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42627 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 53a70bd4899ab7aa137c5ffbd98f8de7, NAME => 'TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 53a70bd4899ab7aa137c5ffbd98f8de7, NAME => 'TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:40:34,509 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/B/78d718ba93ee40c5a5bbd69e83675d39, entries=150, sequenceid=157, filesize=11.9 K 2024-11-22T03:40:34,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,510 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/.tmp/C/602d166562cc4643a27f5cb843e33ddd as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/C/602d166562cc4643a27f5cb843e33ddd 2024-11-22T03:40:34,514 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/C/602d166562cc4643a27f5cb843e33ddd, entries=150, sequenceid=157, filesize=11.9 K 2024-11-22T03:40:34,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,515 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for 53a70bd4899ab7aa137c5ffbd98f8de7 in 1673ms, sequenceid=157, compaction requested=true 2024-11-22T03:40:34,515 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 53a70bd4899ab7aa137c5ffbd98f8de7: 2024-11-22T03:40:34,515 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 53a70bd4899ab7aa137c5ffbd98f8de7:A, priority=-2147483648, current under compaction store size is 1 2024-11-22T03:40:34,515 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T03:40:34,515 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-22T03:40:34,515 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 53a70bd4899ab7aa137c5ffbd98f8de7:B, priority=-2147483648, current under compaction store size is 2 2024-11-22T03:40:34,515 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-22T03:40:34,515 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T03:40:34,515 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 53a70bd4899ab7aa137c5ffbd98f8de7:C, priority=-2147483648, current under compaction store size is 3 2024-11-22T03:40:34,516 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T03:40:34,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,517 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48410 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-22T03:40:34,517 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 141464 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-22T03:40:34,517 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HStore(1541): 53a70bd4899ab7aa137c5ffbd98f8de7/B is initiating minor compaction (all files) 2024-11-22T03:40:34,517 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.HStore(1541): 53a70bd4899ab7aa137c5ffbd98f8de7/A is initiating minor compaction (all files) 2024-11-22T03:40:34,517 INFO [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 53a70bd4899ab7aa137c5ffbd98f8de7/B in TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7. 2024-11-22T03:40:34,517 INFO [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 53a70bd4899ab7aa137c5ffbd98f8de7/A in TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7. 2024-11-22T03:40:34,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,517 INFO [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/A/37890ab5a23441c98c79cd41491f6b91, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/A/3962f084c74946c1a72c72b791c92b7a, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/A/5e32ef50efbc490fa570b73d9c23882d, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/A/c17d0df2f5a94a859b36aecabc975e26] into tmpdir=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/.tmp, totalSize=138.1 K 2024-11-22T03:40:34,517 INFO [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/B/5d02aff14b9c4ee3bab729c0f3a53633, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/B/566cd0fe6c4b4a17b8eb691d57317312, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/B/0eb207b7aae442e8b69682b733b11ef8, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/B/78d718ba93ee40c5a5bbd69e83675d39] into tmpdir=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/.tmp, totalSize=47.3 K 2024-11-22T03:40:34,517 INFO [RS:0;c85114ed5096:42545-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7. 2024-11-22T03:40:34,517 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7. files: [hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/A/37890ab5a23441c98c79cd41491f6b91, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/A/3962f084c74946c1a72c72b791c92b7a, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/A/5e32ef50efbc490fa570b73d9c23882d, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/A/c17d0df2f5a94a859b36aecabc975e26] 2024-11-22T03:40:34,518 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] compactions.Compactor(225): Compacting 5d02aff14b9c4ee3bab729c0f3a53633, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1732246824629 2024-11-22T03:40:34,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,518 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] compactions.Compactor(225): Compacting 37890ab5a23441c98c79cd41491f6b91, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1732246824629 2024-11-22T03:40:34,519 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] compactions.Compactor(225): Compacting 566cd0fe6c4b4a17b8eb691d57317312, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=120, earliestPutTs=1732246826264 2024-11-22T03:40:34,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-22T03:40:34,519 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3962f084c74946c1a72c72b791c92b7a, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=120, earliestPutTs=1732246826264 2024-11-22T03:40:34,519 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] compactions.Compactor(225): Compacting 0eb207b7aae442e8b69682b733b11ef8, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1732246828467 2024-11-22T03:40:34,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,519 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5e32ef50efbc490fa570b73d9c23882d, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1732246828467 2024-11-22T03:40:34,519 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] compactions.Compactor(225): Compacting 78d718ba93ee40c5a5bbd69e83675d39, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1732246830708 2024-11-22T03:40:34,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,520 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] compactions.Compactor(225): Compacting c17d0df2f5a94a859b36aecabc975e26, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1732246830708 2024-11-22T03:40:34,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,533 INFO [RS:0;c85114ed5096:42545-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=53a70bd4899ab7aa137c5ffbd98f8de7] 2024-11-22T03:40:34,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,537 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241122c9055287864b4fdfbc296c71a4f4ef81_53a70bd4899ab7aa137c5ffbd98f8de7 store=[table=TestAcidGuarantees family=A region=53a70bd4899ab7aa137c5ffbd98f8de7] 2024-11-22T03:40:34,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,538 INFO [RS:0;c85114ed5096:42545-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 53a70bd4899ab7aa137c5ffbd98f8de7#B#compaction#147 average throughput is 1.31 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-22T03:40:34,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,538 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/.tmp/B/458453fa85414b57ab33da89d3d353d5 is 50, key is test_row_0/B:col10/1732246832841/Put/seqid=0 2024-11-22T03:40:34,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,544 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241122c9055287864b4fdfbc296c71a4f4ef81_53a70bd4899ab7aa137c5ffbd98f8de7, store=[table=TestAcidGuarantees family=A region=53a70bd4899ab7aa137c5ffbd98f8de7] 2024-11-22T03:40:34,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,544 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241122c9055287864b4fdfbc296c71a4f4ef81_53a70bd4899ab7aa137c5ffbd98f8de7 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=53a70bd4899ab7aa137c5ffbd98f8de7] 2024-11-22T03:40:34,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,575 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073741997_1173 (size=12493) 2024-11-22T03:40:34,580 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073741998_1174 (size=4469) 2024-11-22T03:40:34,583 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/.tmp/B/458453fa85414b57ab33da89d3d353d5 as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/B/458453fa85414b57ab33da89d3d353d5 2024-11-22T03:40:34,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,593 INFO [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 53a70bd4899ab7aa137c5ffbd98f8de7/B of 53a70bd4899ab7aa137c5ffbd98f8de7 into 458453fa85414b57ab33da89d3d353d5(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T03:40:34,593 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 53a70bd4899ab7aa137c5ffbd98f8de7: 2024-11-22T03:40:34,593 INFO [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., storeName=53a70bd4899ab7aa137c5ffbd98f8de7/B, priority=12, startTime=1732246834515; duration=0sec 2024-11-22T03:40:34,593 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T03:40:34,593 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 53a70bd4899ab7aa137c5ffbd98f8de7:B 2024-11-22T03:40:34,593 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-22T03:40:34,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,595 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48410 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-22T03:40:34,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,595 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HStore(1541): 53a70bd4899ab7aa137c5ffbd98f8de7/C is initiating minor compaction (all files) 2024-11-22T03:40:34,595 INFO [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 53a70bd4899ab7aa137c5ffbd98f8de7/C in TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7. 2024-11-22T03:40:34,595 INFO [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/C/88d255ccef494fa28358c67603862c6b, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/C/13ff308f4c924ed097f50cedcf9e4ccb, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/C/5a654c2fcc654d899dbc108782885a6e, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/C/602d166562cc4643a27f5cb843e33ddd] into tmpdir=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/.tmp, totalSize=47.3 K 2024-11-22T03:40:34,595 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] compactions.Compactor(225): Compacting 88d255ccef494fa28358c67603862c6b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1732246824629 2024-11-22T03:40:34,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,596 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] compactions.Compactor(225): Compacting 13ff308f4c924ed097f50cedcf9e4ccb, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=120, earliestPutTs=1732246826264 2024-11-22T03:40:34,596 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] compactions.Compactor(225): Compacting 5a654c2fcc654d899dbc108782885a6e, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1732246828467 2024-11-22T03:40:34,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,596 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] compactions.Compactor(225): Compacting 602d166562cc4643a27f5cb843e33ddd, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1732246830708 2024-11-22T03:40:34,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,610 INFO [RS:0;c85114ed5096:42545-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 53a70bd4899ab7aa137c5ffbd98f8de7#C#compaction#149 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-22T03:40:34,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,610 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/.tmp/C/1f43889e38134100b786865eb95ed73c is 50, key is test_row_0/C:col10/1732246832841/Put/seqid=0 2024-11-22T03:40:34,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,626 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073741999_1175 (size=12493) 2024-11-22T03:40:34,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,640 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/.tmp/C/1f43889e38134100b786865eb95ed73c as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/C/1f43889e38134100b786865eb95ed73c 2024-11-22T03:40:34,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,646 INFO [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 53a70bd4899ab7aa137c5ffbd98f8de7/C of 53a70bd4899ab7aa137c5ffbd98f8de7 into 1f43889e38134100b786865eb95ed73c(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T03:40:34,646 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 53a70bd4899ab7aa137c5ffbd98f8de7: 2024-11-22T03:40:34,646 INFO [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., storeName=53a70bd4899ab7aa137c5ffbd98f8de7/C, priority=12, startTime=1732246834515; duration=0sec 2024-11-22T03:40:34,647 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T03:40:34,647 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 53a70bd4899ab7aa137c5ffbd98f8de7:C 2024-11-22T03:40:34,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,660 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42545 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-22T03:40:34,660 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7. 2024-11-22T03:40:34,660 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2902): Flushing 53a70bd4899ab7aa137c5ffbd98f8de7 3/3 column families, dataSize=46.96 KB heapSize=123.80 KB 2024-11-22T03:40:34,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,679 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112239e616d613ad44fbaf20afe64555a54f_53a70bd4899ab7aa137c5ffbd98f8de7 is 50, key is test_row_1/A:col10/1732246832863/Put/seqid=0 2024-11-22T03:40:34,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,690 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742000_1176 (size=9814) 2024-11-22T03:40:34,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(8855): Flush requested on 53a70bd4899ab7aa137c5ffbd98f8de7 2024-11-22T03:40:34,927 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7. as already flushing 2024-11-22T03:40:34,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:34,981 INFO [RS:0;c85114ed5096:42545-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 53a70bd4899ab7aa137c5ffbd98f8de7#A#compaction#148 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T03:40:34,981 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/.tmp/A/dbaa37aa5655449ebcc5a8d41bdc29b3 is 175, key is test_row_0/A:col10/1732246832841/Put/seqid=0 2024-11-22T03:40:34,985 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742001_1177 (size=31447) 2024-11-22T03:40:35,021 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:35,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34684 deadline: 1732246895019, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:35,022 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:35,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34746 deadline: 1732246895019, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:35,022 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:35,022 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:35,022 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:40:35,022 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to c85114ed5096,42545,1732246786482 for 'test_row_0' in 53a70bd4899ab7aa137c5ffbd98f8de7 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2159 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-22T03:40:35,022 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:35,022 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:35,022 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:40:35,025 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:35,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34754 deadline: 1732246895022, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:35,025 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:35,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34770 deadline: 1732246895022, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:35,025 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:35,025 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:35,025 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:40:35,026 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:35,026 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:35,026 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:40:35,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-22T03:40:35,091 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:35,096 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112239e616d613ad44fbaf20afe64555a54f_53a70bd4899ab7aa137c5ffbd98f8de7 to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112239e616d613ad44fbaf20afe64555a54f_53a70bd4899ab7aa137c5ffbd98f8de7 2024-11-22T03:40:35,097 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/.tmp/A/301aac9f89b0439d9dd315820ee96e9d, store: [table=TestAcidGuarantees family=A region=53a70bd4899ab7aa137c5ffbd98f8de7] 2024-11-22T03:40:35,098 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/.tmp/A/301aac9f89b0439d9dd315820ee96e9d is 175, key is test_row_1/A:col10/1732246832863/Put/seqid=0 2024-11-22T03:40:35,101 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742002_1178 (size=22461) 2024-11-22T03:40:35,130 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:35,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34746 deadline: 1732246895129, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:35,131 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:35,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34754 deadline: 1732246895129, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:35,131 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:35,131 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:35,131 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:40:35,131 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:35,131 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:35,131 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:35,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34770 deadline: 1732246895129, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:35,131 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:40:35,131 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:35,131 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:35,131 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:40:35,341 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:35,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34746 deadline: 1732246895339, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:35,342 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:35,342 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:35,342 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:40:35,342 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:35,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34770 deadline: 1732246895339, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:35,343 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:35,343 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:35,343 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:35,343 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:40:35,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34754 deadline: 1732246895339, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:35,344 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:35,344 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:35,344 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:40:35,391 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/.tmp/A/dbaa37aa5655449ebcc5a8d41bdc29b3 as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/A/dbaa37aa5655449ebcc5a8d41bdc29b3 2024-11-22T03:40:35,397 INFO [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 53a70bd4899ab7aa137c5ffbd98f8de7/A of 53a70bd4899ab7aa137c5ffbd98f8de7 into dbaa37aa5655449ebcc5a8d41bdc29b3(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T03:40:35,397 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 53a70bd4899ab7aa137c5ffbd98f8de7: 2024-11-22T03:40:35,397 INFO [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., storeName=53a70bd4899ab7aa137c5ffbd98f8de7/A, priority=12, startTime=1732246834515; duration=0sec 2024-11-22T03:40:35,397 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T03:40:35,397 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 53a70bd4899ab7aa137c5ffbd98f8de7:A 2024-11-22T03:40:35,502 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=169, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/.tmp/A/301aac9f89b0439d9dd315820ee96e9d 2024-11-22T03:40:35,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/.tmp/B/d7b9030840324a2a802c9612754d5ec5 is 50, key is test_row_1/B:col10/1732246832863/Put/seqid=0 2024-11-22T03:40:35,518 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742003_1179 (size=9757) 2024-11-22T03:40:35,520 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=169 (bloomFilter=true), to=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/.tmp/B/d7b9030840324a2a802c9612754d5ec5 2024-11-22T03:40:35,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/.tmp/C/a97834f8bb5a4a0e943a53f01047faae is 50, key is test_row_1/C:col10/1732246832863/Put/seqid=0 2024-11-22T03:40:35,535 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742004_1180 (size=9757) 2024-11-22T03:40:35,651 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:35,652 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:35,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34746 deadline: 1732246895649, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:35,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34754 deadline: 1732246895649, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:35,652 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:35,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34770 deadline: 1732246895649, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:35,652 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:35,652 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:35,652 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:40:35,653 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:35,653 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:35,653 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:35,653 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:40:35,653 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:35,653 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:40:35,936 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=169 (bloomFilter=true), to=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/.tmp/C/a97834f8bb5a4a0e943a53f01047faae 2024-11-22T03:40:35,942 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/.tmp/A/301aac9f89b0439d9dd315820ee96e9d as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/A/301aac9f89b0439d9dd315820ee96e9d 2024-11-22T03:40:35,949 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/A/301aac9f89b0439d9dd315820ee96e9d, entries=100, sequenceid=169, filesize=21.9 K 2024-11-22T03:40:35,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/.tmp/B/d7b9030840324a2a802c9612754d5ec5 as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/B/d7b9030840324a2a802c9612754d5ec5 2024-11-22T03:40:35,954 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/B/d7b9030840324a2a802c9612754d5ec5, entries=100, sequenceid=169, filesize=9.5 K 2024-11-22T03:40:35,955 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/.tmp/C/a97834f8bb5a4a0e943a53f01047faae as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/C/a97834f8bb5a4a0e943a53f01047faae 2024-11-22T03:40:35,962 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/C/a97834f8bb5a4a0e943a53f01047faae, entries=100, sequenceid=169, filesize=9.5 K 2024-11-22T03:40:35,963 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~123.75 KB/126720, currentSize=161.02 KB/164880 for 53a70bd4899ab7aa137c5ffbd98f8de7 in 1303ms, sequenceid=169, compaction requested=false 2024-11-22T03:40:35,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2603): Flush status journal for 53a70bd4899ab7aa137c5ffbd98f8de7: 2024-11-22T03:40:35,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7. 2024-11-22T03:40:35,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=47 2024-11-22T03:40:35,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42627 {}] master.HMaster(4169): Remote procedure done, pid=47 2024-11-22T03:40:35,967 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=47, resume processing ppid=46 2024-11-22T03:40:35,967 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=47, ppid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0730 sec 2024-11-22T03:40:35,969 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees in 2.0790 sec 2024-11-22T03:40:36,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-22T03:40:36,039 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-22T03:40:36,039 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42627 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-22T03:40:36,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42627 {}] procedure2.ProcedureExecutor(1139): Stored pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees 2024-11-22T03:40:36,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-22T03:40:36,041 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-22T03:40:36,042 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-22T03:40:36,042 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=49, ppid=48, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-22T03:40:36,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-22T03:40:36,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(8855): Flush requested on 53a70bd4899ab7aa137c5ffbd98f8de7 2024-11-22T03:40:36,160 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-22T03:40:36,161 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 53a70bd4899ab7aa137c5ffbd98f8de7 3/3 column families, dataSize=167.72 KB heapSize=440.20 KB 2024-11-22T03:40:36,175 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411229cc84c6925eb42a5bead0bf0bc8b24ca_53a70bd4899ab7aa137c5ffbd98f8de7 is 50, key is test_row_0/A:col10/1732246835021/Put/seqid=0 2024-11-22T03:40:36,178 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:36,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34746 deadline: 1732246896170, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:36,179 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:36,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34754 deadline: 1732246896170, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:36,179 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:36,179 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:36,179 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:40:36,179 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:36,179 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:36,179 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:36,179 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:40:36,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34770 deadline: 1732246896171, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:36,180 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:36,180 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:36,180 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:40:36,184 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742005_1181 (size=14794) 2024-11-22T03:40:36,196 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42545 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-22T03:40:36,196 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7. 2024-11-22T03:40:36,196 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7. as already flushing 2024-11-22T03:40:36,196 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7. 2024-11-22T03:40:36,196 ERROR [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 53a70bd4899ab7aa137c5ffbd98f8de7, NAME => 'TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:40:36,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 53a70bd4899ab7aa137c5ffbd98f8de7, NAME => 'TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:40:36,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42627 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 53a70bd4899ab7aa137c5ffbd98f8de7, NAME => 'TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 53a70bd4899ab7aa137c5ffbd98f8de7, NAME => 'TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:40:36,292 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:36,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34746 deadline: 1732246896289, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:36,293 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:36,293 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:36,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34754 deadline: 1732246896289, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:36,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34770 deadline: 1732246896289, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:36,293 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:36,293 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:36,293 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:40:36,293 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:36,293 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:36,293 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:40:36,293 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:36,293 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:36,293 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:40:36,350 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42545 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-22T03:40:36,350 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7. 2024-11-22T03:40:36,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7. as already flushing 2024-11-22T03:40:36,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7. 2024-11-22T03:40:36,351 ERROR [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 53a70bd4899ab7aa137c5ffbd98f8de7, NAME => 'TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:40:36,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 53a70bd4899ab7aa137c5ffbd98f8de7, NAME => 'TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:40:36,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42627 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 53a70bd4899ab7aa137c5ffbd98f8de7, NAME => 'TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 53a70bd4899ab7aa137c5ffbd98f8de7, NAME => 'TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:40:36,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-22T03:40:36,499 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:36,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34746 deadline: 1732246896499, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:36,499 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:36,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34754 deadline: 1732246896499, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:36,500 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:36,500 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:36,500 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:40:36,500 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:36,500 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:36,500 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:36,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34770 deadline: 1732246896499, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:36,500 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:40:36,500 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:36,500 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:36,500 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:40:36,503 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42545 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-22T03:40:36,503 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7. 2024-11-22T03:40:36,503 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7. as already flushing 2024-11-22T03:40:36,504 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7. 2024-11-22T03:40:36,504 ERROR [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 53a70bd4899ab7aa137c5ffbd98f8de7, NAME => 'TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:40:36,504 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 53a70bd4899ab7aa137c5ffbd98f8de7, NAME => 'TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:40:36,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42627 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 53a70bd4899ab7aa137c5ffbd98f8de7, NAME => 'TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 53a70bd4899ab7aa137c5ffbd98f8de7, NAME => 'TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:40:36,585 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:36,589 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411229cc84c6925eb42a5bead0bf0bc8b24ca_53a70bd4899ab7aa137c5ffbd98f8de7 to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411229cc84c6925eb42a5bead0bf0bc8b24ca_53a70bd4899ab7aa137c5ffbd98f8de7 2024-11-22T03:40:36,590 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/.tmp/A/05ac7e9bf9124651a9481e7a1b8a612c, store: [table=TestAcidGuarantees family=A region=53a70bd4899ab7aa137c5ffbd98f8de7] 2024-11-22T03:40:36,590 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/.tmp/A/05ac7e9bf9124651a9481e7a1b8a612c is 175, key is test_row_0/A:col10/1732246835021/Put/seqid=0 2024-11-22T03:40:36,596 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742006_1182 (size=39749) 2024-11-22T03:40:36,656 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42545 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-22T03:40:36,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7. 2024-11-22T03:40:36,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7. as already flushing 2024-11-22T03:40:36,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7. 2024-11-22T03:40:36,656 ERROR [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 53a70bd4899ab7aa137c5ffbd98f8de7, NAME => 'TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:40:36,657 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 53a70bd4899ab7aa137c5ffbd98f8de7, NAME => 'TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:40:36,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42627 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 53a70bd4899ab7aa137c5ffbd98f8de7, NAME => 'TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 53a70bd4899ab7aa137c5ffbd98f8de7, NAME => 'TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:40:36,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-22T03:40:36,809 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42545 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-22T03:40:36,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7. 2024-11-22T03:40:36,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7. as already flushing 2024-11-22T03:40:36,810 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7. 2024-11-22T03:40:36,810 ERROR [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 53a70bd4899ab7aa137c5ffbd98f8de7, NAME => 'TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:40:36,810 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 53a70bd4899ab7aa137c5ffbd98f8de7, NAME => 'TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:40:36,810 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:36,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34746 deadline: 1732246896809, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:36,810 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:36,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42627 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 53a70bd4899ab7aa137c5ffbd98f8de7, NAME => 'TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 53a70bd4899ab7aa137c5ffbd98f8de7, NAME => 'TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:40:36,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34754 deadline: 1732246896809, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:36,811 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:36,811 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:36,811 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:36,811 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:40:36,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34770 deadline: 1732246896809, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:36,811 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:36,811 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:36,811 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:36,811 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:36,811 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:40:36,811 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:40:36,962 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42545 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-22T03:40:36,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7. 2024-11-22T03:40:36,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7. as already flushing 2024-11-22T03:40:36,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7. 2024-11-22T03:40:36,963 ERROR [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 53a70bd4899ab7aa137c5ffbd98f8de7, NAME => 'TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:40:36,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 53a70bd4899ab7aa137c5ffbd98f8de7, NAME => 'TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:40:36,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42627 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 53a70bd4899ab7aa137c5ffbd98f8de7, NAME => 'TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 53a70bd4899ab7aa137c5ffbd98f8de7, NAME => 'TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:40:36,996 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=198, memsize=55.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/.tmp/A/05ac7e9bf9124651a9481e7a1b8a612c 2024-11-22T03:40:37,004 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/.tmp/B/0839855b903340eea4b225e2cd9f365c is 50, key is test_row_0/B:col10/1732246835021/Put/seqid=0 2024-11-22T03:40:37,008 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742007_1183 (size=12151) 2024-11-22T03:40:37,051 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:37,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34684 deadline: 1732246897049, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:37,051 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to c85114ed5096,42545,1732246786482 for 'test_row_0' in 53a70bd4899ab7aa137c5ffbd98f8de7 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4189 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-22T03:40:37,052 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:37,052 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:37,052 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:40:37,115 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42545 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-22T03:40:37,115 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7. 2024-11-22T03:40:37,116 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7. as already flushing 2024-11-22T03:40:37,116 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7. 2024-11-22T03:40:37,116 ERROR [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 53a70bd4899ab7aa137c5ffbd98f8de7, NAME => 'TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:40:37,116 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 53a70bd4899ab7aa137c5ffbd98f8de7, NAME => 'TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:40:37,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42627 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 53a70bd4899ab7aa137c5ffbd98f8de7, NAME => 'TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 53a70bd4899ab7aa137c5ffbd98f8de7, NAME => 'TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:40:37,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-22T03:40:37,268 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42545 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-22T03:40:37,268 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7. 2024-11-22T03:40:37,268 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7. as already flushing 2024-11-22T03:40:37,268 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7. 2024-11-22T03:40:37,268 ERROR [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 53a70bd4899ab7aa137c5ffbd98f8de7, NAME => 'TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:40:37,268 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 53a70bd4899ab7aa137c5ffbd98f8de7, NAME => 'TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:40:37,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42627 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 53a70bd4899ab7aa137c5ffbd98f8de7, NAME => 'TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 53a70bd4899ab7aa137c5ffbd98f8de7, NAME => 'TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:40:37,322 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:37,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34746 deadline: 1732246897319, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:37,323 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:37,323 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:37,323 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:37,323 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:40:37,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34770 deadline: 1732246897319, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:37,323 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:37,323 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:37,323 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:37,323 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:40:37,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34754 deadline: 1732246897319, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:37,323 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:37,323 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:37,324 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:40:37,409 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=198 (bloomFilter=true), to=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/.tmp/B/0839855b903340eea4b225e2cd9f365c 2024-11-22T03:40:37,416 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/.tmp/C/32449fb167bb4c77b76aec1177511cdd is 50, key is test_row_0/C:col10/1732246835021/Put/seqid=0 2024-11-22T03:40:37,420 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742008_1184 (size=12151) 2024-11-22T03:40:37,421 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42545 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-22T03:40:37,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7. 2024-11-22T03:40:37,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7. as already flushing 2024-11-22T03:40:37,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7. 2024-11-22T03:40:37,421 ERROR [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 53a70bd4899ab7aa137c5ffbd98f8de7, NAME => 'TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:40:37,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 53a70bd4899ab7aa137c5ffbd98f8de7, NAME => 'TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:40:37,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42627 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 53a70bd4899ab7aa137c5ffbd98f8de7, NAME => 'TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 53a70bd4899ab7aa137c5ffbd98f8de7, NAME => 'TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:40:37,574 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42545 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-22T03:40:37,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7. 2024-11-22T03:40:37,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7. as already flushing 2024-11-22T03:40:37,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7. 2024-11-22T03:40:37,574 ERROR [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 53a70bd4899ab7aa137c5ffbd98f8de7, NAME => 'TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:40:37,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 53a70bd4899ab7aa137c5ffbd98f8de7, NAME => 'TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:40:37,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42627 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 53a70bd4899ab7aa137c5ffbd98f8de7, NAME => 'TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 53a70bd4899ab7aa137c5ffbd98f8de7, NAME => 'TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:40:37,727 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42545 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-22T03:40:37,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7. 2024-11-22T03:40:37,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7. as already flushing 2024-11-22T03:40:37,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7. 2024-11-22T03:40:37,727 ERROR [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 53a70bd4899ab7aa137c5ffbd98f8de7, NAME => 'TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:40:37,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 53a70bd4899ab7aa137c5ffbd98f8de7, NAME => 'TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:40:37,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42627 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 53a70bd4899ab7aa137c5ffbd98f8de7, NAME => 'TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 53a70bd4899ab7aa137c5ffbd98f8de7, NAME => 'TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:40:37,821 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=198 (bloomFilter=true), to=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/.tmp/C/32449fb167bb4c77b76aec1177511cdd 2024-11-22T03:40:37,826 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/.tmp/A/05ac7e9bf9124651a9481e7a1b8a612c as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/A/05ac7e9bf9124651a9481e7a1b8a612c 2024-11-22T03:40:37,831 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/A/05ac7e9bf9124651a9481e7a1b8a612c, entries=200, sequenceid=198, filesize=38.8 K 2024-11-22T03:40:37,832 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/.tmp/B/0839855b903340eea4b225e2cd9f365c as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/B/0839855b903340eea4b225e2cd9f365c 2024-11-22T03:40:37,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:37,836 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/B/0839855b903340eea4b225e2cd9f365c, entries=150, sequenceid=198, filesize=11.9 K 2024-11-22T03:40:37,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:37,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:37,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:37,838 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/.tmp/C/32449fb167bb4c77b76aec1177511cdd as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/C/32449fb167bb4c77b76aec1177511cdd 2024-11-22T03:40:37,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:37,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:37,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:37,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:37,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:37,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:37,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:37,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:37,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:37,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:37,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:37,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:37,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:37,842 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/C/32449fb167bb4c77b76aec1177511cdd, entries=150, sequenceid=198, filesize=11.9 K 2024-11-22T03:40:37,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:37,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:37,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:37,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:37,843 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~167.72 KB/171750, heapSize ~440.16 KB/450720, currentSize=40.25 KB/41220 for 53a70bd4899ab7aa137c5ffbd98f8de7 in 1682ms, sequenceid=198, compaction requested=true 2024-11-22T03:40:37,843 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 53a70bd4899ab7aa137c5ffbd98f8de7: 2024-11-22T03:40:37,843 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 53a70bd4899ab7aa137c5ffbd98f8de7:A, priority=-2147483648, current under compaction store size is 1 2024-11-22T03:40:37,843 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T03:40:37,843 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T03:40:37,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:37,843 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 53a70bd4899ab7aa137c5ffbd98f8de7:B, priority=-2147483648, current under compaction store size is 2 2024-11-22T03:40:37,843 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T03:40:37,843 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T03:40:37,843 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 53a70bd4899ab7aa137c5ffbd98f8de7:C, priority=-2147483648, current under compaction store size is 3 2024-11-22T03:40:37,843 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T03:40:37,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:37,845 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 93657 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T03:40:37,845 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.HStore(1541): 53a70bd4899ab7aa137c5ffbd98f8de7/A is initiating minor compaction (all files) 2024-11-22T03:40:37,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:37,846 INFO [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 53a70bd4899ab7aa137c5ffbd98f8de7/A in TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7. 2024-11-22T03:40:37,846 INFO [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/A/dbaa37aa5655449ebcc5a8d41bdc29b3, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/A/301aac9f89b0439d9dd315820ee96e9d, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/A/05ac7e9bf9124651a9481e7a1b8a612c] into tmpdir=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/.tmp, totalSize=91.5 K 2024-11-22T03:40:37,846 INFO [RS:0;c85114ed5096:42545-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7. 2024-11-22T03:40:37,846 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7. files: [hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/A/dbaa37aa5655449ebcc5a8d41bdc29b3, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/A/301aac9f89b0439d9dd315820ee96e9d, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/A/05ac7e9bf9124651a9481e7a1b8a612c] 2024-11-22T03:40:37,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:37,846 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34401 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T03:40:37,846 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HStore(1541): 53a70bd4899ab7aa137c5ffbd98f8de7/B is initiating minor compaction (all files) 2024-11-22T03:40:37,846 INFO [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 53a70bd4899ab7aa137c5ffbd98f8de7/B in TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7. 2024-11-22T03:40:37,846 INFO [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/B/458453fa85414b57ab33da89d3d353d5, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/B/d7b9030840324a2a802c9612754d5ec5, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/B/0839855b903340eea4b225e2cd9f365c] into tmpdir=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/.tmp, totalSize=33.6 K 2024-11-22T03:40:37,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:37,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:37,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:37,848 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] compactions.Compactor(225): Compacting 458453fa85414b57ab33da89d3d353d5, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1732246830708 2024-11-22T03:40:37,848 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] compactions.Compactor(225): Compacting dbaa37aa5655449ebcc5a8d41bdc29b3, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1732246830708 2024-11-22T03:40:37,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:37,849 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] compactions.Compactor(225): Compacting 301aac9f89b0439d9dd315820ee96e9d, keycount=100, bloomtype=ROW, size=21.9 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1732246832862 2024-11-22T03:40:37,849 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] compactions.Compactor(225): Compacting d7b9030840324a2a802c9612754d5ec5, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1732246832862 2024-11-22T03:40:37,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:37,849 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] compactions.Compactor(225): Compacting 05ac7e9bf9124651a9481e7a1b8a612c, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=198, earliestPutTs=1732246834990 2024-11-22T03:40:37,849 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] compactions.Compactor(225): Compacting 0839855b903340eea4b225e2cd9f365c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=198, earliestPutTs=1732246835020 2024-11-22T03:40:37,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:37,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:37,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:37,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:37,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:37,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:37,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:37,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:37,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:37,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:37,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:37,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:37,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:37,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:37,857 INFO [RS:0;c85114ed5096:42545-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=53a70bd4899ab7aa137c5ffbd98f8de7] 2024-11-22T03:40:37,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:37,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:37,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:37,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:37,859 INFO [RS:0;c85114ed5096:42545-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 53a70bd4899ab7aa137c5ffbd98f8de7#B#compaction#157 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-22T03:40:37,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:37,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:37,860 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/.tmp/B/aee8bd82bc0d4137b4688fb8ba92ec4f is 50, key is test_row_0/B:col10/1732246835021/Put/seqid=0 2024-11-22T03:40:37,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:37,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:37,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:37,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:37,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:37,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:37,862 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411224422023880354f48bf95cc41a49eb5da_53a70bd4899ab7aa137c5ffbd98f8de7 store=[table=TestAcidGuarantees family=A region=53a70bd4899ab7aa137c5ffbd98f8de7] 2024-11-22T03:40:37,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:37,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:37,864 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411224422023880354f48bf95cc41a49eb5da_53a70bd4899ab7aa137c5ffbd98f8de7, store=[table=TestAcidGuarantees family=A region=53a70bd4899ab7aa137c5ffbd98f8de7] 2024-11-22T03:40:37,864 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411224422023880354f48bf95cc41a49eb5da_53a70bd4899ab7aa137c5ffbd98f8de7 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=53a70bd4899ab7aa137c5ffbd98f8de7] 2024-11-22T03:40:37,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:37,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:37,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:37,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:37,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:37,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:37,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:37,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:37,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:37,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:37,868 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742009_1185 (size=12595) 2024-11-22T03:40:37,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:37,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:37,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:37,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:37,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:37,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:37,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:37,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:37,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:37,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:37,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:37,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:37,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:37,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:37,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:37,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:37,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:37,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:37,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:37,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:37,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:37,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:37,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:37,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:37,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:37,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:37,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:37,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:37,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:37,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:37,879 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42545 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-22T03:40:37,880 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7. 2024-11-22T03:40:37,880 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2902): Flushing 53a70bd4899ab7aa137c5ffbd98f8de7 3/3 column families, dataSize=40.25 KB heapSize=106.22 KB 2024-11-22T03:40:37,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:37,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:37,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:37,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:37,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:37,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:37,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:37,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:37,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:37,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:37,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:37,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:37,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:37,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:37,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:37,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:37,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:37,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:37,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:37,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:37,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:37,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:37,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:37,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:37,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:37,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:37,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:37,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:37,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:37,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:37,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:37,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:37,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:37,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:37,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:37,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:37,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411222ddd57bc976f42d3b60b21f045a960d7_53a70bd4899ab7aa137c5ffbd98f8de7 is 50, key is test_row_0/A:col10/1732246836169/Put/seqid=0 2024-11-22T03:40:37,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:37,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:37,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:37,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:37,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:37,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:37,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:37,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:37,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:37,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:37,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:37,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:37,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:37,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:37,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:37,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:37,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:37,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:37,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:37,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:37,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:37,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:37,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:37,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:37,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:37,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:37,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:37,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:37,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:37,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:37,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:37,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:37,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:37,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:37,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:37,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:37,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:37,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:37,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:37,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:37,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:37,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:37,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:37,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:37,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:37,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:37,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:37,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:37,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:37,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:37,919 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742010_1186 (size=4469) 2024-11-22T03:40:37,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:37,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:37,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:37,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:37,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:37,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:37,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:37,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:37,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:37,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:37,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:37,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:37,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:37,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:37,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:37,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:37,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:37,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:37,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:37,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:37,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:37,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:37,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:37,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:37,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:37,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:37,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:37,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:37,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:37,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:37,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:37,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:37,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:37,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:37,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:37,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:37,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:37,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:37,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:37,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:37,939 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742011_1187 (size=9814) 2024-11-22T03:40:37,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:37,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:37,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:37,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:37,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:37,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:37,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:37,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:37,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:37,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:37,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:37,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:37,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:37,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:37,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:37,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:37,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:37,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:37,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:37,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:37,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:37,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:37,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:37,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:37,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:37,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:37,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:37,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:37,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:37,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:37,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:37,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:37,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:37,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:37,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:37,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:37,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:37,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:37,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:37,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:37,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:37,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:37,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:37,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:37,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:37,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:37,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:37,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:37,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:37,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:37,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:37,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:37,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:37,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:37,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:37,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:37,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:37,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:37,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:37,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:37,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:37,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:37,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:37,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:37,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:37,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:37,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:37,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:37,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:37,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:37,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:37,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:37,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:37,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:37,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:37,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:37,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:37,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:37,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:37,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:37,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:37,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:37,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:37,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:37,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:37,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:37,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:37,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:37,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:37,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:37,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:37,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:37,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:37,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:37,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:37,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:37,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:37,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:37,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:37,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:37,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:37,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:37,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:37,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:37,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:37,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:37,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:37,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:37,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:37,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:37,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:37,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:37,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:37,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:37,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:37,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:37,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:37,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:37,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:37,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:37,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:37,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:37,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:37,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:37,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:37,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:37,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:37,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:37,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:37,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:37,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:37,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:37,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:37,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:37,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:37,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:37,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:37,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:37,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:37,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:37,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:37,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:37,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:37,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-22T03:40:38,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,277 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/.tmp/B/aee8bd82bc0d4137b4688fb8ba92ec4f as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/B/aee8bd82bc0d4137b4688fb8ba92ec4f 2024-11-22T03:40:38,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,284 INFO [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 53a70bd4899ab7aa137c5ffbd98f8de7/B of 53a70bd4899ab7aa137c5ffbd98f8de7 into aee8bd82bc0d4137b4688fb8ba92ec4f(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T03:40:38,284 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 53a70bd4899ab7aa137c5ffbd98f8de7: 2024-11-22T03:40:38,284 INFO [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., storeName=53a70bd4899ab7aa137c5ffbd98f8de7/B, priority=13, startTime=1732246837843; duration=0sec 2024-11-22T03:40:38,284 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T03:40:38,284 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 53a70bd4899ab7aa137c5ffbd98f8de7:B 2024-11-22T03:40:38,284 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T03:40:38,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,286 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34401 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T03:40:38,286 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HStore(1541): 53a70bd4899ab7aa137c5ffbd98f8de7/C is initiating minor compaction (all files) 2024-11-22T03:40:38,286 INFO [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 53a70bd4899ab7aa137c5ffbd98f8de7/C in TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7. 2024-11-22T03:40:38,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,286 INFO [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/C/1f43889e38134100b786865eb95ed73c, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/C/a97834f8bb5a4a0e943a53f01047faae, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/C/32449fb167bb4c77b76aec1177511cdd] into tmpdir=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/.tmp, totalSize=33.6 K 2024-11-22T03:40:38,287 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] compactions.Compactor(225): Compacting 1f43889e38134100b786865eb95ed73c, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1732246830708 2024-11-22T03:40:38,287 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] compactions.Compactor(225): Compacting a97834f8bb5a4a0e943a53f01047faae, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1732246832862 2024-11-22T03:40:38,287 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] compactions.Compactor(225): Compacting 32449fb167bb4c77b76aec1177511cdd, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=198, earliestPutTs=1732246835020 2024-11-22T03:40:38,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,296 INFO [RS:0;c85114ed5096:42545-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 53a70bd4899ab7aa137c5ffbd98f8de7#C#compaction#159 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-22T03:40:38,297 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/.tmp/C/c2b54d80d34d44d99dd28e1e83f5be24 is 50, key is test_row_0/C:col10/1732246835021/Put/seqid=0 2024-11-22T03:40:38,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,311 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742012_1188 (size=12595) 2024-11-22T03:40:38,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,320 INFO [RS:0;c85114ed5096:42545-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 53a70bd4899ab7aa137c5ffbd98f8de7#A#compaction#156 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T03:40:38,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,321 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/.tmp/A/ee92b1b78bc7449fa42b596839201f90 is 175, key is test_row_0/A:col10/1732246835021/Put/seqid=0 2024-11-22T03:40:38,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,326 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742013_1189 (size=31549) 2024-11-22T03:40:38,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,332 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/.tmp/A/ee92b1b78bc7449fa42b596839201f90 as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/A/ee92b1b78bc7449fa42b596839201f90 2024-11-22T03:40:38,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,338 INFO [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 53a70bd4899ab7aa137c5ffbd98f8de7/A of 53a70bd4899ab7aa137c5ffbd98f8de7 into ee92b1b78bc7449fa42b596839201f90(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T03:40:38,338 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 53a70bd4899ab7aa137c5ffbd98f8de7: 2024-11-22T03:40:38,338 INFO [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., storeName=53a70bd4899ab7aa137c5ffbd98f8de7/A, priority=13, startTime=1732246837843; duration=0sec 2024-11-22T03:40:38,338 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T03:40:38,338 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 53a70bd4899ab7aa137c5ffbd98f8de7:A 2024-11-22T03:40:38,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,345 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411222ddd57bc976f42d3b60b21f045a960d7_53a70bd4899ab7aa137c5ffbd98f8de7 to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411222ddd57bc976f42d3b60b21f045a960d7_53a70bd4899ab7aa137c5ffbd98f8de7 2024-11-22T03:40:38,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(8855): Flush requested on 53a70bd4899ab7aa137c5ffbd98f8de7 2024-11-22T03:40:38,345 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7. as already flushing 2024-11-22T03:40:38,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/.tmp/A/8ee669a9119b4ae0bff2bdcdaf8e75ef, store: [table=TestAcidGuarantees family=A region=53a70bd4899ab7aa137c5ffbd98f8de7] 2024-11-22T03:40:38,347 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/.tmp/A/8ee669a9119b4ae0bff2bdcdaf8e75ef is 175, key is test_row_0/A:col10/1732246836169/Put/seqid=0 2024-11-22T03:40:38,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,375 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742014_1190 (size=22461) 2024-11-22T03:40:38,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:38,404 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:38,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34770 deadline: 1732246898401, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:38,404 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:38,404 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:38,404 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:40:38,409 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:38,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34746 deadline: 1732246898404, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:38,409 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:38,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34754 deadline: 1732246898404, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:38,409 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:38,409 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:38,409 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:40:38,412 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:38,412 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:38,412 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:40:38,508 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:38,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34770 deadline: 1732246898508, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:38,509 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:38,509 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:38,509 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:40:38,521 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:38,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34746 deadline: 1732246898519, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:38,522 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:38,522 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:38,522 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:38,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34754 deadline: 1732246898520, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:38,522 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:40:38,522 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:38,522 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:38,522 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:40:38,717 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/.tmp/C/c2b54d80d34d44d99dd28e1e83f5be24 as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/C/c2b54d80d34d44d99dd28e1e83f5be24 2024-11-22T03:40:38,720 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:38,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34770 deadline: 1732246898718, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:38,720 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:38,720 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:38,720 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:40:38,721 INFO [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 53a70bd4899ab7aa137c5ffbd98f8de7/C of 53a70bd4899ab7aa137c5ffbd98f8de7 into c2b54d80d34d44d99dd28e1e83f5be24(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T03:40:38,722 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 53a70bd4899ab7aa137c5ffbd98f8de7: 2024-11-22T03:40:38,722 INFO [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., storeName=53a70bd4899ab7aa137c5ffbd98f8de7/C, priority=13, startTime=1732246837843; duration=0sec 2024-11-22T03:40:38,722 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T03:40:38,722 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 53a70bd4899ab7aa137c5ffbd98f8de7:C 2024-11-22T03:40:38,730 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:38,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34746 deadline: 1732246898729, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:38,730 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:38,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34754 deadline: 1732246898729, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:38,731 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:38,731 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:38,731 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:38,731 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:38,731 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:40:38,731 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:40:38,776 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=207, memsize=13.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/.tmp/A/8ee669a9119b4ae0bff2bdcdaf8e75ef 2024-11-22T03:40:38,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/.tmp/B/6395bff633da467c93ba2b193e7dc461 is 50, key is test_row_0/B:col10/1732246836169/Put/seqid=0 2024-11-22T03:40:38,790 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742015_1191 (size=9757) 2024-11-22T03:40:39,029 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:39,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34770 deadline: 1732246899028, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:39,030 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:39,030 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:39,030 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:40:39,053 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:39,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34746 deadline: 1732246899038, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:39,054 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:39,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34754 deadline: 1732246899038, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:39,054 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:39,054 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:39,054 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:39,054 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:39,054 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:40:39,054 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:40:39,190 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=207 (bloomFilter=true), to=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/.tmp/B/6395bff633da467c93ba2b193e7dc461 2024-11-22T03:40:39,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/.tmp/C/0d9cacbb17b4405592deccbeb76ad04b is 50, key is test_row_0/C:col10/1732246836169/Put/seqid=0 2024-11-22T03:40:39,212 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742016_1192 (size=9757) 2024-11-22T03:40:39,540 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:39,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34770 deadline: 1732246899539, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:39,541 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:39,541 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:39,541 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:40:39,560 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:39,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34754 deadline: 1732246899558, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:39,560 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:39,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34746 deadline: 1732246899558, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:39,560 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:39,561 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:39,561 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:40:39,561 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:39,561 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:39,561 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:40:39,613 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=207 (bloomFilter=true), to=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/.tmp/C/0d9cacbb17b4405592deccbeb76ad04b 2024-11-22T03:40:39,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/.tmp/A/8ee669a9119b4ae0bff2bdcdaf8e75ef as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/A/8ee669a9119b4ae0bff2bdcdaf8e75ef 2024-11-22T03:40:39,621 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/A/8ee669a9119b4ae0bff2bdcdaf8e75ef, entries=100, sequenceid=207, filesize=21.9 K 2024-11-22T03:40:39,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/.tmp/B/6395bff633da467c93ba2b193e7dc461 as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/B/6395bff633da467c93ba2b193e7dc461 2024-11-22T03:40:39,627 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/B/6395bff633da467c93ba2b193e7dc461, entries=100, sequenceid=207, filesize=9.5 K 2024-11-22T03:40:39,628 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/.tmp/C/0d9cacbb17b4405592deccbeb76ad04b as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/C/0d9cacbb17b4405592deccbeb76ad04b 2024-11-22T03:40:39,632 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/C/0d9cacbb17b4405592deccbeb76ad04b, entries=100, sequenceid=207, filesize=9.5 K 2024-11-22T03:40:39,633 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~106.17 KB/108720, currentSize=161.02 KB/164880 for 53a70bd4899ab7aa137c5ffbd98f8de7 in 1753ms, sequenceid=207, compaction requested=false 2024-11-22T03:40:39,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2603): Flush status journal for 53a70bd4899ab7aa137c5ffbd98f8de7: 2024-11-22T03:40:39,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7. 2024-11-22T03:40:39,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=49 2024-11-22T03:40:39,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42627 {}] master.HMaster(4169): Remote procedure done, pid=49 2024-11-22T03:40:39,636 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=49, resume processing ppid=48 2024-11-22T03:40:39,637 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=49, ppid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.5930 sec 2024-11-22T03:40:39,638 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees in 3.5980 sec 2024-11-22T03:40:40,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-22T03:40:40,208 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-22T03:40:40,209 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42627 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-22T03:40:40,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42627 {}] procedure2.ProcedureExecutor(1139): Stored pid=50, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees 2024-11-22T03:40:40,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-22T03:40:40,211 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=50, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-22T03:40:40,211 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=50, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-22T03:40:40,211 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=51, ppid=50, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-22T03:40:40,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-22T03:40:40,363 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42545 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-22T03:40:40,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7. 2024-11-22T03:40:40,364 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2902): Flushing 53a70bd4899ab7aa137c5ffbd98f8de7 3/3 column families, dataSize=161.02 KB heapSize=422.63 KB 2024-11-22T03:40:40,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411220ec380228b764111a4aedb27ada478af_53a70bd4899ab7aa137c5ffbd98f8de7 is 50, key is test_row_0/A:col10/1732246838388/Put/seqid=0 2024-11-22T03:40:40,375 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742017_1193 (size=12304) 2024-11-22T03:40:40,376 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:40,383 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411220ec380228b764111a4aedb27ada478af_53a70bd4899ab7aa137c5ffbd98f8de7 to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411220ec380228b764111a4aedb27ada478af_53a70bd4899ab7aa137c5ffbd98f8de7 2024-11-22T03:40:40,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/.tmp/A/209e23f980274144b8c85443220b81b4, store: [table=TestAcidGuarantees family=A region=53a70bd4899ab7aa137c5ffbd98f8de7] 2024-11-22T03:40:40,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/.tmp/A/209e23f980274144b8c85443220b81b4 is 175, key is test_row_0/A:col10/1732246838388/Put/seqid=0 2024-11-22T03:40:40,388 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742018_1194 (size=31105) 2024-11-22T03:40:40,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-22T03:40:40,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(8855): Flush requested on 53a70bd4899ab7aa137c5ffbd98f8de7 2024-11-22T03:40:40,549 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7. as already flushing 2024-11-22T03:40:40,562 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:40,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34770 deadline: 1732246900560, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:40,562 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:40,563 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:40,563 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:40:40,571 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:40,571 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:40,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34754 deadline: 1732246900568, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:40,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34746 deadline: 1732246900568, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:40,571 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to c85114ed5096,42545,1732246786482 for 'test_row_1' in 53a70bd4899ab7aa137c5ffbd98f8de7 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2167 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-22T03:40:40,572 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:40,572 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:40,571 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to c85114ed5096,42545,1732246786482 for 'test_row_0' in 53a70bd4899ab7aa137c5ffbd98f8de7 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2167 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-22T03:40:40,572 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:40:40,572 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:40,572 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:40,572 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:40:40,671 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:40,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34770 deadline: 1732246900668, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:40,672 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:40,672 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:40,672 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:40:40,789 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=237, memsize=53.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/.tmp/A/209e23f980274144b8c85443220b81b4 2024-11-22T03:40:40,800 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/.tmp/B/03bcdd7997e24ed8a9d757ac67773e34 is 50, key is test_row_0/B:col10/1732246838388/Put/seqid=0 2024-11-22T03:40:40,803 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742019_1195 (size=12151) 2024-11-22T03:40:40,808 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=237 (bloomFilter=true), to=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/.tmp/B/03bcdd7997e24ed8a9d757ac67773e34 2024-11-22T03:40:40,831 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/.tmp/C/88ca6e8880414276a5133cd009fe1542 is 50, key is test_row_0/C:col10/1732246838388/Put/seqid=0 2024-11-22T03:40:40,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-22T03:40:40,842 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742020_1196 (size=12151) 2024-11-22T03:40:40,880 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:40,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34770 deadline: 1732246900878, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:40,881 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:40,881 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:40,881 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:40:41,090 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:41,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34684 deadline: 1732246901089, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:41,091 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to c85114ed5096,42545,1732246786482 for 'test_row_0' in 53a70bd4899ab7aa137c5ffbd98f8de7 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8228 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-22T03:40:41,091 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:41,091 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:41,091 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:40:41,192 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:41,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34770 deadline: 1732246901189, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:41,193 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:41,193 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:41,193 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:40:41,244 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=237 (bloomFilter=true), to=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/.tmp/C/88ca6e8880414276a5133cd009fe1542 2024-11-22T03:40:41,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/.tmp/A/209e23f980274144b8c85443220b81b4 as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/A/209e23f980274144b8c85443220b81b4 2024-11-22T03:40:41,253 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/A/209e23f980274144b8c85443220b81b4, entries=150, sequenceid=237, filesize=30.4 K 2024-11-22T03:40:41,254 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/.tmp/B/03bcdd7997e24ed8a9d757ac67773e34 as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/B/03bcdd7997e24ed8a9d757ac67773e34 2024-11-22T03:40:41,261 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/B/03bcdd7997e24ed8a9d757ac67773e34, entries=150, sequenceid=237, filesize=11.9 K 2024-11-22T03:40:41,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/.tmp/C/88ca6e8880414276a5133cd009fe1542 as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/C/88ca6e8880414276a5133cd009fe1542 2024-11-22T03:40:41,269 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/C/88ca6e8880414276a5133cd009fe1542, entries=150, sequenceid=237, filesize=11.9 K 2024-11-22T03:40:41,270 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~422.58 KB/432720, currentSize=40.25 KB/41220 for 53a70bd4899ab7aa137c5ffbd98f8de7 in 906ms, sequenceid=237, compaction requested=true 2024-11-22T03:40:41,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2603): Flush status journal for 53a70bd4899ab7aa137c5ffbd98f8de7: 2024-11-22T03:40:41,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7. 2024-11-22T03:40:41,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=51 2024-11-22T03:40:41,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42627 {}] master.HMaster(4169): Remote procedure done, pid=51 2024-11-22T03:40:41,273 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=51, resume processing ppid=50 2024-11-22T03:40:41,273 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=51, ppid=50, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0610 sec 2024-11-22T03:40:41,274 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=50, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees in 1.0640 sec 2024-11-22T03:40:41,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-22T03:40:41,349 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-22T03:40:41,349 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42627 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-22T03:40:41,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42627 {}] procedure2.ProcedureExecutor(1139): Stored pid=52, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees 2024-11-22T03:40:41,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-22T03:40:41,351 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=52, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-22T03:40:41,352 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=52, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-22T03:40:41,352 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=53, ppid=52, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-22T03:40:41,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-22T03:40:41,496 INFO [Thread-671 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-671. 2024-11-22T03:40:41,496 DEBUG [Thread-671 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-22T03:40:41,496 DEBUG [Thread-671 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T03:40:41,496 DEBUG [Thread-671 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T03:40:41,496 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-22T03:40:41,497 INFO [Thread-673 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-673. 2024-11-22T03:40:41,497 DEBUG [Thread-673 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-22T03:40:41,497 DEBUG [Thread-673 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T03:40:41,497 DEBUG [Thread-673 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T03:40:41,497 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-22T03:40:41,500 INFO [Thread-677 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-677. 2024-11-22T03:40:41,500 DEBUG [Thread-677 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-22T03:40:41,500 DEBUG [Thread-677 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T03:40:41,500 DEBUG [Thread-677 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T03:40:41,500 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-22T03:40:41,501 INFO [Thread-675 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-675. 2024-11-22T03:40:41,501 DEBUG [Thread-675 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-22T03:40:41,501 DEBUG [Thread-675 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T03:40:41,501 DEBUG [Thread-675 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T03:40:41,501 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-22T03:40:41,505 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42545 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-22T03:40:41,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7. 2024-11-22T03:40:41,505 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2902): Flushing 53a70bd4899ab7aa137c5ffbd98f8de7 3/3 column families, dataSize=40.25 KB heapSize=106.22 KB 2024-11-22T03:40:41,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241122a7288cad18194fd2855b59327310e24b_53a70bd4899ab7aa137c5ffbd98f8de7 is 50, key is test_row_0/A:col10/1732246840550/Put/seqid=0 2024-11-22T03:40:41,514 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742021_1197 (size=12304) 2024-11-22T03:40:41,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-22T03:40:41,701 INFO [Thread-662 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-662. 2024-11-22T03:40:41,701 DEBUG [Thread-662 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-22T03:40:41,701 DEBUG [Thread-662 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T03:40:41,701 DEBUG [Thread-662 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T03:40:41,701 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-22T03:40:41,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:41,919 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241122a7288cad18194fd2855b59327310e24b_53a70bd4899ab7aa137c5ffbd98f8de7 to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241122a7288cad18194fd2855b59327310e24b_53a70bd4899ab7aa137c5ffbd98f8de7 2024-11-22T03:40:41,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/.tmp/A/d2682200828c491dbe08528025b8fef4, store: [table=TestAcidGuarantees family=A region=53a70bd4899ab7aa137c5ffbd98f8de7] 2024-11-22T03:40:41,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/.tmp/A/d2682200828c491dbe08528025b8fef4 is 175, key is test_row_0/A:col10/1732246840550/Put/seqid=0 2024-11-22T03:40:41,925 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742022_1198 (size=31105) 2024-11-22T03:40:41,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-22T03:40:42,327 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=246, memsize=13.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/.tmp/A/d2682200828c491dbe08528025b8fef4 2024-11-22T03:40:42,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/.tmp/B/5189e20c64924b66b7635f0157948e8d is 50, key is test_row_0/B:col10/1732246840550/Put/seqid=0 2024-11-22T03:40:42,344 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742023_1199 (size=12151) 2024-11-22T03:40:42,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-22T03:40:42,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(8855): Flush requested on 53a70bd4899ab7aa137c5ffbd98f8de7 2024-11-22T03:40:42,592 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7. as already flushing 2024-11-22T03:40:42,593 INFO [Thread-664 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-664. 2024-11-22T03:40:42,593 DEBUG [Thread-664 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-22T03:40:42,593 DEBUG [Thread-664 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T03:40:42,593 DEBUG [Thread-664 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T03:40:42,593 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-22T03:40:42,602 INFO [Thread-666 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-666. 2024-11-22T03:40:42,602 DEBUG [Thread-666 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-22T03:40:42,602 DEBUG [Thread-666 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T03:40:42,602 DEBUG [Thread-666 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T03:40:42,602 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-22T03:40:42,745 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=246 (bloomFilter=true), to=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/.tmp/B/5189e20c64924b66b7635f0157948e8d 2024-11-22T03:40:42,757 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/.tmp/C/126194a8ab064b90a292ea3dc37e3f9e is 50, key is test_row_0/C:col10/1732246840550/Put/seqid=0 2024-11-22T03:40:42,761 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742024_1200 (size=12151) 2024-11-22T03:40:42,935 INFO [Thread-668 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-668. 2024-11-22T03:40:42,935 DEBUG [Thread-668 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-22T03:40:42,935 DEBUG [Thread-668 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T03:40:42,936 DEBUG [Thread-668 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T03:40:42,936 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-22T03:40:43,162 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=246 (bloomFilter=true), to=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/.tmp/C/126194a8ab064b90a292ea3dc37e3f9e 2024-11-22T03:40:43,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/.tmp/A/d2682200828c491dbe08528025b8fef4 as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/A/d2682200828c491dbe08528025b8fef4 2024-11-22T03:40:43,178 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/A/d2682200828c491dbe08528025b8fef4, entries=150, sequenceid=246, filesize=30.4 K 2024-11-22T03:40:43,180 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/.tmp/B/5189e20c64924b66b7635f0157948e8d as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/B/5189e20c64924b66b7635f0157948e8d 2024-11-22T03:40:43,184 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/B/5189e20c64924b66b7635f0157948e8d, entries=150, sequenceid=246, filesize=11.9 K 2024-11-22T03:40:43,185 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/.tmp/C/126194a8ab064b90a292ea3dc37e3f9e as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/C/126194a8ab064b90a292ea3dc37e3f9e 2024-11-22T03:40:43,188 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/C/126194a8ab064b90a292ea3dc37e3f9e, entries=150, sequenceid=246, filesize=11.9 K 2024-11-22T03:40:43,189 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~106.17 KB/108720, currentSize=26.84 KB/27480 for 53a70bd4899ab7aa137c5ffbd98f8de7 in 1684ms, sequenceid=246, compaction requested=true 2024-11-22T03:40:43,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2603): Flush status journal for 53a70bd4899ab7aa137c5ffbd98f8de7: 2024-11-22T03:40:43,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7. 2024-11-22T03:40:43,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=53 2024-11-22T03:40:43,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42627 {}] master.HMaster(4169): Remote procedure done, pid=53 2024-11-22T03:40:43,192 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=53, resume processing ppid=52 2024-11-22T03:40:43,192 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=53, ppid=52, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8380 sec 2024-11-22T03:40:43,193 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=52, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees in 1.8420 sec 2024-11-22T03:40:43,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-22T03:40:43,499 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-22T03:40:44,822 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-22T03:40:49,529 DEBUG [regionserver/c85114ed5096:0.Chore.1 {}] throttle.PressureAwareCompactionThroughputController(103): CompactionPressure is 0.07692307692307693, tune throughput to 53.85 MB/second 2024-11-22T03:40:51,121 INFO [Thread-660 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-660. 2024-11-22T03:40:51,121 DEBUG [Thread-660 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-22T03:40:51,121 DEBUG [Thread-660 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T03:40:51,121 DEBUG [Thread-660 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T03:40:51,121 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-22T03:40:51,122 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-22T03:40:51,122 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 28 2024-11-22T03:40:51,122 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 30 2024-11-22T03:40:51,122 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 25 2024-11-22T03:40:51,122 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 97 2024-11-22T03:40:51,122 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 16 2024-11-22T03:40:51,122 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-22T03:40:51,122 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4111 2024-11-22T03:40:51,122 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3893 2024-11-22T03:40:51,122 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-22T03:40:51,122 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1709 2024-11-22T03:40:51,122 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 5127 rows 2024-11-22T03:40:51,122 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1707 2024-11-22T03:40:51,122 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 5118 rows 2024-11-22T03:40:51,122 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-22T03:40:51,122 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobMixedAtomicity(AcidGuaranteesTestBase.java:132) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-22T03:40:51,122 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T03:40:51,123 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T03:40:51,123 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-22T03:40:51,124 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42627 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-22T03:40:51,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42627 {}] procedure2.ProcedureExecutor(1139): Stored pid=54, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-22T03:40:51,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-22T03:40:51,130 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732246851130"}]},"ts":"1732246851130"} 2024-11-22T03:40:51,133 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-22T03:40:51,133 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-22T03:40:51,134 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=55, ppid=54, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-22T03:40:51,136 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=56, ppid=55, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=53a70bd4899ab7aa137c5ffbd98f8de7, UNASSIGN}] 2024-11-22T03:40:51,136 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=56, ppid=55, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=53a70bd4899ab7aa137c5ffbd98f8de7, UNASSIGN 2024-11-22T03:40:51,138 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=56 updating hbase:meta row=53a70bd4899ab7aa137c5ffbd98f8de7, regionState=CLOSING, regionLocation=c85114ed5096,42545,1732246786482 2024-11-22T03:40:51,139 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=56, ppid=55, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=53a70bd4899ab7aa137c5ffbd98f8de7, UNASSIGN because future has completed 2024-11-22T03:40:51,139 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-22T03:40:51,139 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=57, ppid=56, state=RUNNABLE, hasLock=false; CloseRegionProcedure 53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482}] 2024-11-22T03:40:51,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-22T03:40:51,292 INFO [RS_CLOSE_REGION-regionserver/c85114ed5096:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] handler.UnassignRegionHandler(122): Close 53a70bd4899ab7aa137c5ffbd98f8de7 2024-11-22T03:40:51,292 DEBUG [RS_CLOSE_REGION-regionserver/c85114ed5096:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-22T03:40:51,292 DEBUG [RS_CLOSE_REGION-regionserver/c85114ed5096:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegion(1722): Closing 53a70bd4899ab7aa137c5ffbd98f8de7, disabling compactions & flushes 2024-11-22T03:40:51,292 INFO [RS_CLOSE_REGION-regionserver/c85114ed5096:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7. 2024-11-22T03:40:51,293 DEBUG [RS_CLOSE_REGION-regionserver/c85114ed5096:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7. 2024-11-22T03:40:51,293 DEBUG [RS_CLOSE_REGION-regionserver/c85114ed5096:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7. after waiting 0 ms 2024-11-22T03:40:51,293 DEBUG [RS_CLOSE_REGION-regionserver/c85114ed5096:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7. 2024-11-22T03:40:51,293 INFO [RS_CLOSE_REGION-regionserver/c85114ed5096:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegion(2902): Flushing 53a70bd4899ab7aa137c5ffbd98f8de7 3/3 column families, dataSize=33.54 KB heapSize=88.64 KB 2024-11-22T03:40:51,303 DEBUG [RS_CLOSE_REGION-regionserver/c85114ed5096:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112289751bfbbca747daa3bb607d4e9d6076_53a70bd4899ab7aa137c5ffbd98f8de7 is 50, key is test_row_0/A:col10/1732246851119/Put/seqid=0 2024-11-22T03:40:51,306 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742025_1201 (size=9814) 2024-11-22T03:40:51,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-22T03:40:51,708 DEBUG [RS_CLOSE_REGION-regionserver/c85114ed5096:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:51,717 INFO [RS_CLOSE_REGION-regionserver/c85114ed5096:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112289751bfbbca747daa3bb607d4e9d6076_53a70bd4899ab7aa137c5ffbd98f8de7 to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112289751bfbbca747daa3bb607d4e9d6076_53a70bd4899ab7aa137c5ffbd98f8de7 2024-11-22T03:40:51,718 DEBUG [RS_CLOSE_REGION-regionserver/c85114ed5096:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/.tmp/A/31efff59c5a345b0b5a2f8ef120267c5, store: [table=TestAcidGuarantees family=A region=53a70bd4899ab7aa137c5ffbd98f8de7] 2024-11-22T03:40:51,719 DEBUG [RS_CLOSE_REGION-regionserver/c85114ed5096:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/.tmp/A/31efff59c5a345b0b5a2f8ef120267c5 is 175, key is test_row_0/A:col10/1732246851119/Put/seqid=0 2024-11-22T03:40:51,723 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742026_1202 (size=22461) 2024-11-22T03:40:51,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-22T03:40:52,124 INFO [RS_CLOSE_REGION-regionserver/c85114ed5096:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=254, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/.tmp/A/31efff59c5a345b0b5a2f8ef120267c5 2024-11-22T03:40:52,135 DEBUG [RS_CLOSE_REGION-regionserver/c85114ed5096:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/.tmp/B/9561430e74ae4835916ab106071089d4 is 50, key is test_row_0/B:col10/1732246851119/Put/seqid=0 2024-11-22T03:40:52,139 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742027_1203 (size=9757) 2024-11-22T03:40:52,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-22T03:40:52,540 INFO [RS_CLOSE_REGION-regionserver/c85114ed5096:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=254 (bloomFilter=true), to=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/.tmp/B/9561430e74ae4835916ab106071089d4 2024-11-22T03:40:52,548 DEBUG [RS_CLOSE_REGION-regionserver/c85114ed5096:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/.tmp/C/67eb775366634627b571dbe017f27bde is 50, key is test_row_0/C:col10/1732246851119/Put/seqid=0 2024-11-22T03:40:52,551 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742028_1204 (size=9757) 2024-11-22T03:40:52,953 INFO [RS_CLOSE_REGION-regionserver/c85114ed5096:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=254 (bloomFilter=true), to=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/.tmp/C/67eb775366634627b571dbe017f27bde 2024-11-22T03:40:52,964 DEBUG [RS_CLOSE_REGION-regionserver/c85114ed5096:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/.tmp/A/31efff59c5a345b0b5a2f8ef120267c5 as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/A/31efff59c5a345b0b5a2f8ef120267c5 2024-11-22T03:40:52,971 INFO [RS_CLOSE_REGION-regionserver/c85114ed5096:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/A/31efff59c5a345b0b5a2f8ef120267c5, entries=100, sequenceid=254, filesize=21.9 K 2024-11-22T03:40:52,973 DEBUG [RS_CLOSE_REGION-regionserver/c85114ed5096:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/.tmp/B/9561430e74ae4835916ab106071089d4 as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/B/9561430e74ae4835916ab106071089d4 2024-11-22T03:40:52,979 INFO [RS_CLOSE_REGION-regionserver/c85114ed5096:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/B/9561430e74ae4835916ab106071089d4, entries=100, sequenceid=254, filesize=9.5 K 2024-11-22T03:40:52,980 DEBUG [RS_CLOSE_REGION-regionserver/c85114ed5096:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/.tmp/C/67eb775366634627b571dbe017f27bde as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/C/67eb775366634627b571dbe017f27bde 2024-11-22T03:40:52,985 INFO [RS_CLOSE_REGION-regionserver/c85114ed5096:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/C/67eb775366634627b571dbe017f27bde, entries=100, sequenceid=254, filesize=9.5 K 2024-11-22T03:40:52,986 INFO [RS_CLOSE_REGION-regionserver/c85114ed5096:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for 53a70bd4899ab7aa137c5ffbd98f8de7 in 1693ms, sequenceid=254, compaction requested=true 2024-11-22T03:40:52,987 DEBUG [StoreCloser-TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/A/adbcfb9d053f42d09e39c8f01d7386e5, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/A/e5deced64e254ca1990394b04a60031b, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/A/d8349d7cb4304958b4fc704955f33898, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/A/11fcb99717c4471d9f90041c22e98b38, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/A/c250eb0bdb3442a6a4d45a5802f450d5, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/A/6caebab891ba492e96765f14a60ef5ec, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/A/37890ab5a23441c98c79cd41491f6b91, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/A/3962f084c74946c1a72c72b791c92b7a, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/A/5e32ef50efbc490fa570b73d9c23882d, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/A/c17d0df2f5a94a859b36aecabc975e26, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/A/dbaa37aa5655449ebcc5a8d41bdc29b3, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/A/301aac9f89b0439d9dd315820ee96e9d, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/A/05ac7e9bf9124651a9481e7a1b8a612c] to archive 2024-11-22T03:40:52,988 DEBUG [StoreCloser-TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-22T03:40:52,990 DEBUG [StoreCloser-TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/A/adbcfb9d053f42d09e39c8f01d7386e5 to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/A/adbcfb9d053f42d09e39c8f01d7386e5 2024-11-22T03:40:52,992 DEBUG [StoreCloser-TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/A/e5deced64e254ca1990394b04a60031b to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/A/e5deced64e254ca1990394b04a60031b 2024-11-22T03:40:52,994 DEBUG [StoreCloser-TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/A/d8349d7cb4304958b4fc704955f33898 to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/A/d8349d7cb4304958b4fc704955f33898 2024-11-22T03:40:52,995 DEBUG [StoreCloser-TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/A/11fcb99717c4471d9f90041c22e98b38 to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/A/11fcb99717c4471d9f90041c22e98b38 2024-11-22T03:40:52,997 DEBUG [StoreCloser-TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/A/c250eb0bdb3442a6a4d45a5802f450d5 to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/A/c250eb0bdb3442a6a4d45a5802f450d5 2024-11-22T03:40:52,999 DEBUG [StoreCloser-TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/A/6caebab891ba492e96765f14a60ef5ec to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/A/6caebab891ba492e96765f14a60ef5ec 2024-11-22T03:40:53,001 DEBUG [StoreCloser-TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/A/37890ab5a23441c98c79cd41491f6b91 to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/A/37890ab5a23441c98c79cd41491f6b91 2024-11-22T03:40:53,003 DEBUG [StoreCloser-TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/A/3962f084c74946c1a72c72b791c92b7a to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/A/3962f084c74946c1a72c72b791c92b7a 2024-11-22T03:40:53,005 DEBUG [StoreCloser-TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/A/5e32ef50efbc490fa570b73d9c23882d to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/A/5e32ef50efbc490fa570b73d9c23882d 2024-11-22T03:40:53,007 DEBUG [StoreCloser-TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/A/c17d0df2f5a94a859b36aecabc975e26 to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/A/c17d0df2f5a94a859b36aecabc975e26 2024-11-22T03:40:53,009 DEBUG [StoreCloser-TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/A/dbaa37aa5655449ebcc5a8d41bdc29b3 to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/A/dbaa37aa5655449ebcc5a8d41bdc29b3 2024-11-22T03:40:53,011 DEBUG [StoreCloser-TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/A/301aac9f89b0439d9dd315820ee96e9d to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/A/301aac9f89b0439d9dd315820ee96e9d 2024-11-22T03:40:53,012 DEBUG [StoreCloser-TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/A/05ac7e9bf9124651a9481e7a1b8a612c to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/A/05ac7e9bf9124651a9481e7a1b8a612c 2024-11-22T03:40:53,014 DEBUG [StoreCloser-TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/B/78300911a140463dbffb11243c5aac63, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/B/6e91529c14514ad7a28fdc83c85bec71, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/B/0fe94b926f4b4fa2a0d02cc23e244525, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/B/5591c9c3e4004cbaac9e8867ef88ff7c, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/B/01854badc16c4a9191303248880dda95, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/B/5d02aff14b9c4ee3bab729c0f3a53633, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/B/0d78c199960542dcb3920cc0be2e9e3a, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/B/566cd0fe6c4b4a17b8eb691d57317312, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/B/0eb207b7aae442e8b69682b733b11ef8, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/B/458453fa85414b57ab33da89d3d353d5, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/B/78d718ba93ee40c5a5bbd69e83675d39, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/B/d7b9030840324a2a802c9612754d5ec5, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/B/0839855b903340eea4b225e2cd9f365c] to archive 2024-11-22T03:40:53,014 DEBUG [StoreCloser-TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-22T03:40:53,016 DEBUG [StoreCloser-TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/B/78300911a140463dbffb11243c5aac63 to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/B/78300911a140463dbffb11243c5aac63 2024-11-22T03:40:53,017 DEBUG [StoreCloser-TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/B/6e91529c14514ad7a28fdc83c85bec71 to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/B/6e91529c14514ad7a28fdc83c85bec71 2024-11-22T03:40:53,018 DEBUG [StoreCloser-TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/B/0fe94b926f4b4fa2a0d02cc23e244525 to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/B/0fe94b926f4b4fa2a0d02cc23e244525 2024-11-22T03:40:53,019 DEBUG [StoreCloser-TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/B/5591c9c3e4004cbaac9e8867ef88ff7c to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/B/5591c9c3e4004cbaac9e8867ef88ff7c 2024-11-22T03:40:53,020 DEBUG [StoreCloser-TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/B/01854badc16c4a9191303248880dda95 to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/B/01854badc16c4a9191303248880dda95 2024-11-22T03:40:53,021 DEBUG [StoreCloser-TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/B/5d02aff14b9c4ee3bab729c0f3a53633 to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/B/5d02aff14b9c4ee3bab729c0f3a53633 2024-11-22T03:40:53,022 DEBUG [StoreCloser-TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/B/0d78c199960542dcb3920cc0be2e9e3a to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/B/0d78c199960542dcb3920cc0be2e9e3a 2024-11-22T03:40:53,022 DEBUG [StoreCloser-TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/B/566cd0fe6c4b4a17b8eb691d57317312 to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/B/566cd0fe6c4b4a17b8eb691d57317312 2024-11-22T03:40:53,023 DEBUG [StoreCloser-TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/B/0eb207b7aae442e8b69682b733b11ef8 to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/B/0eb207b7aae442e8b69682b733b11ef8 2024-11-22T03:40:53,024 DEBUG [StoreCloser-TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/B/458453fa85414b57ab33da89d3d353d5 to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/B/458453fa85414b57ab33da89d3d353d5 2024-11-22T03:40:53,026 DEBUG [StoreCloser-TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/B/78d718ba93ee40c5a5bbd69e83675d39 to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/B/78d718ba93ee40c5a5bbd69e83675d39 2024-11-22T03:40:53,027 DEBUG [StoreCloser-TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/B/d7b9030840324a2a802c9612754d5ec5 to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/B/d7b9030840324a2a802c9612754d5ec5 2024-11-22T03:40:53,028 DEBUG [StoreCloser-TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/B/0839855b903340eea4b225e2cd9f365c to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/B/0839855b903340eea4b225e2cd9f365c 2024-11-22T03:40:53,030 DEBUG [StoreCloser-TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/C/23755d88fe7a49a39fab68586618038f, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/C/4305809f7be94277adba899314ed55e5, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/C/f935efd1741a4353b24f1b2e16390aa2, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/C/7f3b2c7d4c3946629603539a7af7692f, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/C/ae9be35fafae4f56ab4c55048983720a, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/C/88d255ccef494fa28358c67603862c6b, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/C/ea6858231cd142fb95b139d52dac81e1, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/C/13ff308f4c924ed097f50cedcf9e4ccb, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/C/5a654c2fcc654d899dbc108782885a6e, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/C/1f43889e38134100b786865eb95ed73c, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/C/602d166562cc4643a27f5cb843e33ddd, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/C/a97834f8bb5a4a0e943a53f01047faae, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/C/32449fb167bb4c77b76aec1177511cdd] to archive 2024-11-22T03:40:53,030 DEBUG [StoreCloser-TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-22T03:40:53,032 DEBUG [StoreCloser-TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/C/23755d88fe7a49a39fab68586618038f to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/C/23755d88fe7a49a39fab68586618038f 2024-11-22T03:40:53,033 DEBUG [StoreCloser-TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/C/4305809f7be94277adba899314ed55e5 to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/C/4305809f7be94277adba899314ed55e5 2024-11-22T03:40:53,034 DEBUG [StoreCloser-TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/C/f935efd1741a4353b24f1b2e16390aa2 to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/C/f935efd1741a4353b24f1b2e16390aa2 2024-11-22T03:40:53,036 DEBUG [StoreCloser-TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/C/7f3b2c7d4c3946629603539a7af7692f to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/C/7f3b2c7d4c3946629603539a7af7692f 2024-11-22T03:40:53,037 DEBUG [StoreCloser-TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/C/ae9be35fafae4f56ab4c55048983720a to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/C/ae9be35fafae4f56ab4c55048983720a 2024-11-22T03:40:53,039 DEBUG [StoreCloser-TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/C/88d255ccef494fa28358c67603862c6b to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/C/88d255ccef494fa28358c67603862c6b 2024-11-22T03:40:53,040 DEBUG [StoreCloser-TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/C/ea6858231cd142fb95b139d52dac81e1 to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/C/ea6858231cd142fb95b139d52dac81e1 2024-11-22T03:40:53,041 DEBUG [StoreCloser-TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/C/13ff308f4c924ed097f50cedcf9e4ccb to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/C/13ff308f4c924ed097f50cedcf9e4ccb 2024-11-22T03:40:53,043 DEBUG [StoreCloser-TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/C/5a654c2fcc654d899dbc108782885a6e to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/C/5a654c2fcc654d899dbc108782885a6e 2024-11-22T03:40:53,045 DEBUG [StoreCloser-TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/C/1f43889e38134100b786865eb95ed73c to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/C/1f43889e38134100b786865eb95ed73c 2024-11-22T03:40:53,047 DEBUG [StoreCloser-TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/C/602d166562cc4643a27f5cb843e33ddd to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/C/602d166562cc4643a27f5cb843e33ddd 2024-11-22T03:40:53,048 DEBUG [StoreCloser-TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/C/a97834f8bb5a4a0e943a53f01047faae to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/C/a97834f8bb5a4a0e943a53f01047faae 2024-11-22T03:40:53,049 DEBUG [StoreCloser-TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/C/32449fb167bb4c77b76aec1177511cdd to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/C/32449fb167bb4c77b76aec1177511cdd 2024-11-22T03:40:53,053 DEBUG [RS_CLOSE_REGION-regionserver/c85114ed5096:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/recovered.edits/257.seqid, newMaxSeqId=257, maxSeqId=4 2024-11-22T03:40:53,053 INFO [RS_CLOSE_REGION-regionserver/c85114ed5096:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7. 2024-11-22T03:40:53,054 DEBUG [RS_CLOSE_REGION-regionserver/c85114ed5096:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegion(1676): Region close journal for 53a70bd4899ab7aa137c5ffbd98f8de7: Waiting for close lock at 1732246851292Running coprocessor pre-close hooks at 1732246851292Disabling compacts and flushes for region at 1732246851292Disabling writes for close at 1732246851293 (+1 ms)Obtaining lock to block concurrent updates at 1732246851293Preparing flush snapshotting stores in 53a70bd4899ab7aa137c5ffbd98f8de7 at 1732246851293Finished memstore snapshotting TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=90720, getOffHeapSize=0, getCellsCount=750 at 1732246851294 (+1 ms)Flushing stores of TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7. at 1732246851295 (+1 ms)Flushing 53a70bd4899ab7aa137c5ffbd98f8de7/A: creating writer at 1732246851295Flushing mob file 53a70bd4899ab7aa137c5ffbd98f8de7/A: appending metadata at 1732246851303 (+8 ms)Flushing mob file 53a70bd4899ab7aa137c5ffbd98f8de7/A: closing flushed file at 1732246851303Flushing 53a70bd4899ab7aa137c5ffbd98f8de7/A: appending metadata at 1732246851718 (+415 ms)Flushing 53a70bd4899ab7aa137c5ffbd98f8de7/A: closing flushed file at 1732246851718Flushing 53a70bd4899ab7aa137c5ffbd98f8de7/B: creating writer at 1732246852131 (+413 ms)Flushing 53a70bd4899ab7aa137c5ffbd98f8de7/B: appending metadata at 1732246852134 (+3 ms)Flushing 53a70bd4899ab7aa137c5ffbd98f8de7/B: closing flushed file at 1732246852134Flushing 53a70bd4899ab7aa137c5ffbd98f8de7/C: creating writer at 1732246852545 (+411 ms)Flushing 53a70bd4899ab7aa137c5ffbd98f8de7/C: appending metadata at 1732246852547 (+2 ms)Flushing 53a70bd4899ab7aa137c5ffbd98f8de7/C: closing flushed file at 1732246852548 (+1 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@cc6b201: reopening flushed file at 1732246852962 (+414 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1b04556f: reopening flushed file at 1732246852971 (+9 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@51daada8: reopening flushed file at 1732246852979 (+8 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for 53a70bd4899ab7aa137c5ffbd98f8de7 in 1693ms, sequenceid=254, compaction requested=true at 1732246852986 (+7 ms)Writing region close event to WAL at 1732246853050 (+64 ms)Running coprocessor post-close hooks at 1732246853053 (+3 ms)Closed at 1732246853053 2024-11-22T03:40:53,055 INFO [RS_CLOSE_REGION-regionserver/c85114ed5096:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] handler.UnassignRegionHandler(157): Closed 53a70bd4899ab7aa137c5ffbd98f8de7 2024-11-22T03:40:53,055 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=56 updating hbase:meta row=53a70bd4899ab7aa137c5ffbd98f8de7, regionState=CLOSED 2024-11-22T03:40:53,056 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=57, ppid=56, state=RUNNABLE, hasLock=false; CloseRegionProcedure 53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 because future has completed 2024-11-22T03:40:53,058 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=57, resume processing ppid=56 2024-11-22T03:40:53,058 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=57, ppid=56, state=SUCCESS, hasLock=false; CloseRegionProcedure 53a70bd4899ab7aa137c5ffbd98f8de7, server=c85114ed5096,42545,1732246786482 in 1.9180 sec 2024-11-22T03:40:53,060 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=56, resume processing ppid=55 2024-11-22T03:40:53,060 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=56, ppid=55, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=53a70bd4899ab7aa137c5ffbd98f8de7, UNASSIGN in 1.9220 sec 2024-11-22T03:40:53,062 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=55, resume processing ppid=54 2024-11-22T03:40:53,062 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=55, ppid=54, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.9260 sec 2024-11-22T03:40:53,063 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732246853063"}]},"ts":"1732246853063"} 2024-11-22T03:40:53,064 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-22T03:40:53,065 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-22T03:40:53,067 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=54, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.9410 sec 2024-11-22T03:40:53,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-22T03:40:53,290 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-22T03:40:53,291 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42627 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-22T03:40:53,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42627 {}] procedure2.ProcedureExecutor(1139): Stored pid=58, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-22T03:40:53,295 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=58, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-22T03:40:53,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-22T03:40:53,297 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=58, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-22T03:40:53,301 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7 2024-11-22T03:40:53,304 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/A, FileablePath, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/B, FileablePath, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/C, FileablePath, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/recovered.edits] 2024-11-22T03:40:53,308 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/A/209e23f980274144b8c85443220b81b4 to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/A/209e23f980274144b8c85443220b81b4 2024-11-22T03:40:53,310 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/A/31efff59c5a345b0b5a2f8ef120267c5 to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/A/31efff59c5a345b0b5a2f8ef120267c5 2024-11-22T03:40:53,311 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/A/8ee669a9119b4ae0bff2bdcdaf8e75ef to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/A/8ee669a9119b4ae0bff2bdcdaf8e75ef 2024-11-22T03:40:53,313 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/A/d2682200828c491dbe08528025b8fef4 to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/A/d2682200828c491dbe08528025b8fef4 2024-11-22T03:40:53,314 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/A/ee92b1b78bc7449fa42b596839201f90 to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/A/ee92b1b78bc7449fa42b596839201f90 2024-11-22T03:40:53,316 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/B/03bcdd7997e24ed8a9d757ac67773e34 to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/B/03bcdd7997e24ed8a9d757ac67773e34 2024-11-22T03:40:53,317 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/B/5189e20c64924b66b7635f0157948e8d to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/B/5189e20c64924b66b7635f0157948e8d 2024-11-22T03:40:53,318 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/B/6395bff633da467c93ba2b193e7dc461 to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/B/6395bff633da467c93ba2b193e7dc461 2024-11-22T03:40:53,319 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/B/9561430e74ae4835916ab106071089d4 to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/B/9561430e74ae4835916ab106071089d4 2024-11-22T03:40:53,320 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/B/aee8bd82bc0d4137b4688fb8ba92ec4f to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/B/aee8bd82bc0d4137b4688fb8ba92ec4f 2024-11-22T03:40:53,322 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/C/0d9cacbb17b4405592deccbeb76ad04b to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/C/0d9cacbb17b4405592deccbeb76ad04b 2024-11-22T03:40:53,323 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/C/126194a8ab064b90a292ea3dc37e3f9e to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/C/126194a8ab064b90a292ea3dc37e3f9e 2024-11-22T03:40:53,324 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/C/67eb775366634627b571dbe017f27bde to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/C/67eb775366634627b571dbe017f27bde 2024-11-22T03:40:53,325 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/C/88ca6e8880414276a5133cd009fe1542 to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/C/88ca6e8880414276a5133cd009fe1542 2024-11-22T03:40:53,327 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/C/c2b54d80d34d44d99dd28e1e83f5be24 to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/C/c2b54d80d34d44d99dd28e1e83f5be24 2024-11-22T03:40:53,331 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/recovered.edits/257.seqid to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7/recovered.edits/257.seqid 2024-11-22T03:40:53,331 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/53a70bd4899ab7aa137c5ffbd98f8de7 2024-11-22T03:40:53,331 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-22T03:40:53,332 DEBUG [PEWorker-4 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-22T03:40:53,333 DEBUG [PEWorker-4 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-22T03:40:53,338 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411220ec380228b764111a4aedb27ada478af_53a70bd4899ab7aa137c5ffbd98f8de7 to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411220ec380228b764111a4aedb27ada478af_53a70bd4899ab7aa137c5ffbd98f8de7 2024-11-22T03:40:53,339 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411222ddd57bc976f42d3b60b21f045a960d7_53a70bd4899ab7aa137c5ffbd98f8de7 to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411222ddd57bc976f42d3b60b21f045a960d7_53a70bd4899ab7aa137c5ffbd98f8de7 2024-11-22T03:40:53,340 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112239e616d613ad44fbaf20afe64555a54f_53a70bd4899ab7aa137c5ffbd98f8de7 to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112239e616d613ad44fbaf20afe64555a54f_53a70bd4899ab7aa137c5ffbd98f8de7 2024-11-22T03:40:53,341 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112263131f685b454a0da2e7ae7daa54ab61_53a70bd4899ab7aa137c5ffbd98f8de7 to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112263131f685b454a0da2e7ae7daa54ab61_53a70bd4899ab7aa137c5ffbd98f8de7 2024-11-22T03:40:53,342 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411226554a47af350405e9eca14c72b988e3e_53a70bd4899ab7aa137c5ffbd98f8de7 to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411226554a47af350405e9eca14c72b988e3e_53a70bd4899ab7aa137c5ffbd98f8de7 2024-11-22T03:40:53,344 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411228748b81d00cb412aa945bc57a31502db_53a70bd4899ab7aa137c5ffbd98f8de7 to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411228748b81d00cb412aa945bc57a31502db_53a70bd4899ab7aa137c5ffbd98f8de7 2024-11-22T03:40:53,345 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112289751bfbbca747daa3bb607d4e9d6076_53a70bd4899ab7aa137c5ffbd98f8de7 to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112289751bfbbca747daa3bb607d4e9d6076_53a70bd4899ab7aa137c5ffbd98f8de7 2024-11-22T03:40:53,346 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112289917da1df434f2ab99e6a32b9b95d6a_53a70bd4899ab7aa137c5ffbd98f8de7 to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112289917da1df434f2ab99e6a32b9b95d6a_53a70bd4899ab7aa137c5ffbd98f8de7 2024-11-22T03:40:53,347 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411229748ceaee6bd49abb1f4c678ddb5f0fb_53a70bd4899ab7aa137c5ffbd98f8de7 to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411229748ceaee6bd49abb1f4c678ddb5f0fb_53a70bd4899ab7aa137c5ffbd98f8de7 2024-11-22T03:40:53,349 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411229cc84c6925eb42a5bead0bf0bc8b24ca_53a70bd4899ab7aa137c5ffbd98f8de7 to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411229cc84c6925eb42a5bead0bf0bc8b24ca_53a70bd4899ab7aa137c5ffbd98f8de7 2024-11-22T03:40:53,350 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241122a7288cad18194fd2855b59327310e24b_53a70bd4899ab7aa137c5ffbd98f8de7 to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241122a7288cad18194fd2855b59327310e24b_53a70bd4899ab7aa137c5ffbd98f8de7 2024-11-22T03:40:53,351 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241122e5d254e71603421aabcc41d74b4d41e4_53a70bd4899ab7aa137c5ffbd98f8de7 to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241122e5d254e71603421aabcc41d74b4d41e4_53a70bd4899ab7aa137c5ffbd98f8de7 2024-11-22T03:40:53,352 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241122eee8581ccae84b03b9dd79de682c167d_53a70bd4899ab7aa137c5ffbd98f8de7 to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241122eee8581ccae84b03b9dd79de682c167d_53a70bd4899ab7aa137c5ffbd98f8de7 2024-11-22T03:40:53,354 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241122f72aae3cca274599b4c692314ac4b893_53a70bd4899ab7aa137c5ffbd98f8de7 to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241122f72aae3cca274599b4c692314ac4b893_53a70bd4899ab7aa137c5ffbd98f8de7 2024-11-22T03:40:53,354 DEBUG [PEWorker-4 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-22T03:40:53,356 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=58, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-22T03:40:53,359 WARN [PEWorker-4 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-22T03:40:53,361 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-22T03:40:53,362 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=58, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-22T03:40:53,362 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-22T03:40:53,362 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732246853362"}]},"ts":"9223372036854775807"} 2024-11-22T03:40:53,365 INFO [PEWorker-4 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-22T03:40:53,365 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 53a70bd4899ab7aa137c5ffbd98f8de7, NAME => 'TestAcidGuarantees,,1732246818199.53a70bd4899ab7aa137c5ffbd98f8de7.', STARTKEY => '', ENDKEY => ''}] 2024-11-22T03:40:53,365 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-22T03:40:53,365 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732246853365"}]},"ts":"9223372036854775807"} 2024-11-22T03:40:53,368 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-22T03:40:53,369 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(133): Finished pid=58, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-22T03:40:53,370 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=58, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 78 msec 2024-11-22T03:40:53,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-22T03:40:53,409 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-22T03:40:53,410 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-22T03:40:53,420 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testMobMixedAtomicity Thread=238 (was 239), OpenFileDescriptor=449 (was 459), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=378 (was 446), ProcessCount=11 (was 11), AvailableMemoryMB=3193 (was 3560) 2024-11-22T03:40:53,428 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testGetAtomicity Thread=238, OpenFileDescriptor=449, MaxFileDescriptor=1048576, SystemLoadAverage=378, ProcessCount=11, AvailableMemoryMB=3192 2024-11-22T03:40:53,429 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42627 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-22T03:40:53,429 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42627 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-22T03:40:53,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42627 {}] procedure2.ProcedureExecutor(1139): Stored pid=59, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-22T03:40:53,431 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=59, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-22T03:40:53,431 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:53,431 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42627 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 59 2024-11-22T03:40:53,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=59 2024-11-22T03:40:53,432 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=59, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-22T03:40:53,437 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742029_1205 (size=959) 2024-11-22T03:40:53,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=59 2024-11-22T03:40:53,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=59 2024-11-22T03:40:53,844 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => f0af25cc84372b1efb01f18522e2d562, NAME => 'TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a 2024-11-22T03:40:53,856 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742030_1206 (size=53) 2024-11-22T03:40:54,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=59 2024-11-22T03:40:54,259 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-22T03:40:54,259 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing f0af25cc84372b1efb01f18522e2d562, disabling compactions & flushes 2024-11-22T03:40:54,259 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562. 2024-11-22T03:40:54,260 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562. 2024-11-22T03:40:54,260 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562. after waiting 0 ms 2024-11-22T03:40:54,260 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562. 2024-11-22T03:40:54,260 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562. 2024-11-22T03:40:54,260 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for f0af25cc84372b1efb01f18522e2d562: Waiting for close lock at 1732246854259Disabling compacts and flushes for region at 1732246854259Disabling writes for close at 1732246854260 (+1 ms)Writing region close event to WAL at 1732246854260Closed at 1732246854260 2024-11-22T03:40:54,262 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=59, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-22T03:40:54,263 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732246854263"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732246854263"}]},"ts":"1732246854263"} 2024-11-22T03:40:54,266 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-22T03:40:54,266 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=59, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-22T03:40:54,267 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732246854266"}]},"ts":"1732246854266"} 2024-11-22T03:40:54,268 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-22T03:40:54,268 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=60, ppid=59, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f0af25cc84372b1efb01f18522e2d562, ASSIGN}] 2024-11-22T03:40:54,269 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=60, ppid=59, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f0af25cc84372b1efb01f18522e2d562, ASSIGN 2024-11-22T03:40:54,270 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=60, ppid=59, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=f0af25cc84372b1efb01f18522e2d562, ASSIGN; state=OFFLINE, location=c85114ed5096,42545,1732246786482; forceNewPlan=false, retain=false 2024-11-22T03:40:54,421 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=60 updating hbase:meta row=f0af25cc84372b1efb01f18522e2d562, regionState=OPENING, regionLocation=c85114ed5096,42545,1732246786482 2024-11-22T03:40:54,424 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=60, ppid=59, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=f0af25cc84372b1efb01f18522e2d562, ASSIGN because future has completed 2024-11-22T03:40:54,424 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=61, ppid=60, state=RUNNABLE, hasLock=false; OpenRegionProcedure f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482}] 2024-11-22T03:40:54,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=59 2024-11-22T03:40:54,580 INFO [RS_OPEN_REGION-regionserver/c85114ed5096:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562. 2024-11-22T03:40:54,580 DEBUG [RS_OPEN_REGION-regionserver/c85114ed5096:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(7752): Opening region: {ENCODED => f0af25cc84372b1efb01f18522e2d562, NAME => 'TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562.', STARTKEY => '', ENDKEY => ''} 2024-11-22T03:40:54,580 DEBUG [RS_OPEN_REGION-regionserver/c85114ed5096:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees f0af25cc84372b1efb01f18522e2d562 2024-11-22T03:40:54,580 DEBUG [RS_OPEN_REGION-regionserver/c85114ed5096:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-22T03:40:54,580 DEBUG [RS_OPEN_REGION-regionserver/c85114ed5096:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(7794): checking encryption for f0af25cc84372b1efb01f18522e2d562 2024-11-22T03:40:54,580 DEBUG [RS_OPEN_REGION-regionserver/c85114ed5096:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(7797): checking classloading for f0af25cc84372b1efb01f18522e2d562 2024-11-22T03:40:54,582 INFO [StoreOpener-f0af25cc84372b1efb01f18522e2d562-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region f0af25cc84372b1efb01f18522e2d562 2024-11-22T03:40:54,584 INFO [StoreOpener-f0af25cc84372b1efb01f18522e2d562-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f0af25cc84372b1efb01f18522e2d562 columnFamilyName A 2024-11-22T03:40:54,584 DEBUG [StoreOpener-f0af25cc84372b1efb01f18522e2d562-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:54,584 INFO [StoreOpener-f0af25cc84372b1efb01f18522e2d562-1 {}] regionserver.HStore(327): Store=f0af25cc84372b1efb01f18522e2d562/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-22T03:40:54,585 INFO [StoreOpener-f0af25cc84372b1efb01f18522e2d562-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region f0af25cc84372b1efb01f18522e2d562 2024-11-22T03:40:54,586 INFO [StoreOpener-f0af25cc84372b1efb01f18522e2d562-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f0af25cc84372b1efb01f18522e2d562 columnFamilyName B 2024-11-22T03:40:54,586 DEBUG [StoreOpener-f0af25cc84372b1efb01f18522e2d562-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:54,587 INFO [StoreOpener-f0af25cc84372b1efb01f18522e2d562-1 {}] regionserver.HStore(327): Store=f0af25cc84372b1efb01f18522e2d562/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-22T03:40:54,587 INFO [StoreOpener-f0af25cc84372b1efb01f18522e2d562-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region f0af25cc84372b1efb01f18522e2d562 2024-11-22T03:40:54,589 INFO [StoreOpener-f0af25cc84372b1efb01f18522e2d562-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f0af25cc84372b1efb01f18522e2d562 columnFamilyName C 2024-11-22T03:40:54,589 DEBUG [StoreOpener-f0af25cc84372b1efb01f18522e2d562-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:40:54,590 INFO [StoreOpener-f0af25cc84372b1efb01f18522e2d562-1 {}] regionserver.HStore(327): Store=f0af25cc84372b1efb01f18522e2d562/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-22T03:40:54,590 DEBUG [RS_OPEN_REGION-regionserver/c85114ed5096:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(1038): replaying wal for f0af25cc84372b1efb01f18522e2d562 2024-11-22T03:40:54,591 DEBUG [RS_OPEN_REGION-regionserver/c85114ed5096:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562 2024-11-22T03:40:54,592 DEBUG [RS_OPEN_REGION-regionserver/c85114ed5096:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562 2024-11-22T03:40:54,593 DEBUG [RS_OPEN_REGION-regionserver/c85114ed5096:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(1048): stopping wal replay for f0af25cc84372b1efb01f18522e2d562 2024-11-22T03:40:54,593 DEBUG [RS_OPEN_REGION-regionserver/c85114ed5096:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(1060): Cleaning up temporary data for f0af25cc84372b1efb01f18522e2d562 2024-11-22T03:40:54,594 DEBUG [RS_OPEN_REGION-regionserver/c85114ed5096:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-22T03:40:54,595 DEBUG [RS_OPEN_REGION-regionserver/c85114ed5096:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(1093): writing seq id for f0af25cc84372b1efb01f18522e2d562 2024-11-22T03:40:54,597 DEBUG [RS_OPEN_REGION-regionserver/c85114ed5096:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-22T03:40:54,598 INFO [RS_OPEN_REGION-regionserver/c85114ed5096:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(1114): Opened f0af25cc84372b1efb01f18522e2d562; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69466302, jitterRate=0.03512856364250183}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-22T03:40:54,598 DEBUG [RS_OPEN_REGION-regionserver/c85114ed5096:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(1122): Running coprocessor post-open hooks for f0af25cc84372b1efb01f18522e2d562 2024-11-22T03:40:54,599 DEBUG [RS_OPEN_REGION-regionserver/c85114ed5096:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(1006): Region open journal for f0af25cc84372b1efb01f18522e2d562: Running coprocessor pre-open hook at 1732246854580Writing region info on filesystem at 1732246854580Initializing all the Stores at 1732246854582 (+2 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732246854582Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732246854582Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732246854582Cleaning up temporary data from old regions at 1732246854593 (+11 ms)Running coprocessor post-open hooks at 1732246854598 (+5 ms)Region opened successfully at 1732246854599 (+1 ms) 2024-11-22T03:40:54,600 INFO [RS_OPEN_REGION-regionserver/c85114ed5096:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., pid=61, masterSystemTime=1732246854576 2024-11-22T03:40:54,602 DEBUG [RS_OPEN_REGION-regionserver/c85114ed5096:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562. 2024-11-22T03:40:54,602 INFO [RS_OPEN_REGION-regionserver/c85114ed5096:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562. 2024-11-22T03:40:54,603 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=60 updating hbase:meta row=f0af25cc84372b1efb01f18522e2d562, regionState=OPEN, openSeqNum=2, regionLocation=c85114ed5096,42545,1732246786482 2024-11-22T03:40:54,604 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=61, ppid=60, state=RUNNABLE, hasLock=false; OpenRegionProcedure f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 because future has completed 2024-11-22T03:40:54,608 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=61, resume processing ppid=60 2024-11-22T03:40:54,608 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 in 181 msec 2024-11-22T03:40:54,612 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=60, resume processing ppid=59 2024-11-22T03:40:54,612 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=60, ppid=59, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f0af25cc84372b1efb01f18522e2d562, ASSIGN in 340 msec 2024-11-22T03:40:54,613 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=59, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-22T03:40:54,613 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732246854613"}]},"ts":"1732246854613"} 2024-11-22T03:40:54,615 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-22T03:40:54,616 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=59, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-22T03:40:54,617 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=59, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1870 sec 2024-11-22T03:40:55,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=59 2024-11-22T03:40:55,579 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-22T03:40:55,580 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@e04d2b7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T03:40:55,580 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c85114ed5096,42627,-1 for getting cluster id 2024-11-22T03:40:55,580 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-22T03:40:55,581 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '67035b7f-c649-44a8-9db7-17adff49cedb' 2024-11-22T03:40:55,581 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-22T03:40:55,581 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "67035b7f-c649-44a8-9db7-17adff49cedb" 2024-11-22T03:40:55,581 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7c5f4922, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T03:40:55,581 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c85114ed5096,42627,-1] 2024-11-22T03:40:55,582 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-22T03:40:55,582 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T03:40:55,582 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55946, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-22T03:40:55,583 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@215aaae, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T03:40:55,583 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-22T03:40:55,584 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c85114ed5096,42545,1732246786482, seqNum=-1] 2024-11-22T03:40:55,584 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-22T03:40:55,585 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39142, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-22T03:40:55,585 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-22T03:40:55,586 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncConnectionImpl(321): The fetched master address is c85114ed5096,42627,1732246785796 2024-11-22T03:40:55,586 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@75ea09e6 2024-11-22T03:40:55,586 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-22T03:40:55,587 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55950, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-22T03:40:55,588 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@391cff66, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T03:40:55,588 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c85114ed5096,42627,-1 for getting cluster id 2024-11-22T03:40:55,588 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-22T03:40:55,588 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '67035b7f-c649-44a8-9db7-17adff49cedb' 2024-11-22T03:40:55,589 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-22T03:40:55,589 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "67035b7f-c649-44a8-9db7-17adff49cedb" 2024-11-22T03:40:55,589 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@b19e392, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T03:40:55,589 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c85114ed5096,42627,-1] 2024-11-22T03:40:55,589 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-22T03:40:55,589 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T03:40:55,590 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55970, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-22T03:40:55,590 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@468a4b4f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T03:40:55,591 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@135bd0ea, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T03:40:55,591 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c85114ed5096,42627,-1 for getting cluster id 2024-11-22T03:40:55,591 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-22T03:40:55,592 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '67035b7f-c649-44a8-9db7-17adff49cedb' 2024-11-22T03:40:55,592 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-22T03:40:55,592 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "67035b7f-c649-44a8-9db7-17adff49cedb" 2024-11-22T03:40:55,592 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@449a9b8c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T03:40:55,592 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c85114ed5096,42627,-1] 2024-11-22T03:40:55,592 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-22T03:40:55,592 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T03:40:55,593 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55982, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-22T03:40:55,593 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@22056d92, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T03:40:55,593 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4609cead, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T03:40:55,593 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c85114ed5096,42627,-1 for getting cluster id 2024-11-22T03:40:55,594 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-22T03:40:55,594 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '67035b7f-c649-44a8-9db7-17adff49cedb' 2024-11-22T03:40:55,594 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-22T03:40:55,594 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "67035b7f-c649-44a8-9db7-17adff49cedb" 2024-11-22T03:40:55,594 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@550017c0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T03:40:55,594 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c85114ed5096,42627,-1] 2024-11-22T03:40:55,594 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-22T03:40:55,595 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T03:40:55,595 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55986, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-22T03:40:55,596 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@79400ec6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T03:40:55,596 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3a2df097, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T03:40:55,596 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c85114ed5096,42627,-1 for getting cluster id 2024-11-22T03:40:55,597 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-22T03:40:55,597 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '67035b7f-c649-44a8-9db7-17adff49cedb' 2024-11-22T03:40:55,597 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-22T03:40:55,597 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "67035b7f-c649-44a8-9db7-17adff49cedb" 2024-11-22T03:40:55,598 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4a5eeceb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T03:40:55,598 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c85114ed5096,42627,-1] 2024-11-22T03:40:55,598 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-22T03:40:55,598 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T03:40:55,599 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56012, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-22T03:40:55,599 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1bdc066e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T03:40:55,601 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2856abc4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T03:40:55,601 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c85114ed5096,42627,-1 for getting cluster id 2024-11-22T03:40:55,601 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-22T03:40:55,602 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '67035b7f-c649-44a8-9db7-17adff49cedb' 2024-11-22T03:40:55,602 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-22T03:40:55,602 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "67035b7f-c649-44a8-9db7-17adff49cedb" 2024-11-22T03:40:55,603 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@75840309, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T03:40:55,603 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c85114ed5096,42627,-1] 2024-11-22T03:40:55,603 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-22T03:40:55,603 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T03:40:55,604 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56026, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-22T03:40:55,604 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@13d5507d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T03:40:55,605 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7a4f6020, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T03:40:55,605 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c85114ed5096,42627,-1 for getting cluster id 2024-11-22T03:40:55,605 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-22T03:40:55,606 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '67035b7f-c649-44a8-9db7-17adff49cedb' 2024-11-22T03:40:55,606 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-22T03:40:55,606 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "67035b7f-c649-44a8-9db7-17adff49cedb" 2024-11-22T03:40:55,606 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@e56efd6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T03:40:55,606 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c85114ed5096,42627,-1] 2024-11-22T03:40:55,606 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-22T03:40:55,607 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T03:40:55,607 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56046, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-22T03:40:55,608 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@adacdde, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T03:40:55,608 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@692cba37, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T03:40:55,609 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c85114ed5096,42627,-1 for getting cluster id 2024-11-22T03:40:55,609 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-22T03:40:55,609 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '67035b7f-c649-44a8-9db7-17adff49cedb' 2024-11-22T03:40:55,610 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-22T03:40:55,610 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "67035b7f-c649-44a8-9db7-17adff49cedb" 2024-11-22T03:40:55,610 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5316b491, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T03:40:55,610 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c85114ed5096,42627,-1] 2024-11-22T03:40:55,610 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-22T03:40:55,610 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T03:40:55,611 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56064, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-22T03:40:55,611 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2e04f72e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T03:40:55,612 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1d805c88, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T03:40:55,612 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c85114ed5096,42627,-1 for getting cluster id 2024-11-22T03:40:55,612 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-22T03:40:55,613 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '67035b7f-c649-44a8-9db7-17adff49cedb' 2024-11-22T03:40:55,613 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-22T03:40:55,613 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "67035b7f-c649-44a8-9db7-17adff49cedb" 2024-11-22T03:40:55,613 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4299b44f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T03:40:55,613 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c85114ed5096,42627,-1] 2024-11-22T03:40:55,613 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-22T03:40:55,614 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T03:40:55,614 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56080, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-22T03:40:55,615 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@60c7587a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T03:40:55,616 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4861c2d6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T03:40:55,616 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c85114ed5096,42627,-1 for getting cluster id 2024-11-22T03:40:55,616 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-22T03:40:55,616 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '67035b7f-c649-44a8-9db7-17adff49cedb' 2024-11-22T03:40:55,616 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-22T03:40:55,617 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "67035b7f-c649-44a8-9db7-17adff49cedb" 2024-11-22T03:40:55,617 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3eaf1951, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T03:40:55,617 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c85114ed5096,42627,-1] 2024-11-22T03:40:55,617 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-22T03:40:55,617 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T03:40:55,618 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56100, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-22T03:40:55,618 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6bbbe838, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T03:40:55,619 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3397c776, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T03:40:55,619 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c85114ed5096,42627,-1 for getting cluster id 2024-11-22T03:40:55,619 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-22T03:40:55,619 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '67035b7f-c649-44a8-9db7-17adff49cedb' 2024-11-22T03:40:55,620 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-22T03:40:55,620 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "67035b7f-c649-44a8-9db7-17adff49cedb" 2024-11-22T03:40:55,620 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@603eb82e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T03:40:55,620 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c85114ed5096,42627,-1] 2024-11-22T03:40:55,620 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-22T03:40:55,620 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T03:40:55,621 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56116, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-22T03:40:55,621 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@68cd640f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T03:40:55,622 DEBUG [Thread-1004 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-22T03:40:55,622 DEBUG [Thread-993 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-22T03:40:55,623 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c85114ed5096,42545,1732246786482, seqNum=-1] 2024-11-22T03:40:55,623 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c85114ed5096,42545,1732246786482, seqNum=-1] 2024-11-22T03:40:55,623 DEBUG [Thread-998 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-22T03:40:55,623 DEBUG [Thread-996 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-22T03:40:55,623 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-22T03:40:55,623 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-22T03:40:55,623 DEBUG [Thread-1002 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-22T03:40:55,623 DEBUG [Thread-998 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c85114ed5096,42545,1732246786482, seqNum=-1] 2024-11-22T03:40:55,623 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42627 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-22T03:40:55,624 DEBUG [Thread-1000 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-22T03:40:55,624 DEBUG [Thread-998 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-22T03:40:55,624 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c85114ed5096,42545,1732246786482, seqNum=-1] 2024-11-22T03:40:55,624 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c85114ed5096,42545,1732246786482, seqNum=-1] 2024-11-22T03:40:55,624 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-22T03:40:55,624 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-22T03:40:55,624 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39154, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-22T03:40:55,624 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39156, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-22T03:40:55,625 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39172, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-22T03:40:55,625 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c85114ed5096,42545,1732246786482, seqNum=-1] 2024-11-22T03:40:55,625 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39176, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-22T03:40:55,625 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-22T03:40:55,626 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39192, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-22T03:40:55,626 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2] 2024-11-22T03:40:55,626 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2] 2024-11-22T03:40:55,627 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2] 2024-11-22T03:40:55,627 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39208, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-22T03:40:55,628 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2] 2024-11-22T03:40:55,628 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2] 2024-11-22T03:40:55,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42627 {}] procedure2.ProcedureExecutor(1139): Stored pid=62, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees 2024-11-22T03:40:55,629 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2] 2024-11-22T03:40:55,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-22T03:40:55,630 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=62, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-22T03:40:55,631 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=62, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-22T03:40:55,631 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=63, ppid=62, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-22T03:40:55,641 DEBUG [Thread-987 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-22T03:40:55,642 DEBUG [Thread-987 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c85114ed5096,42545,1732246786482, seqNum=-1] 2024-11-22T03:40:55,643 DEBUG [Thread-987 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-22T03:40:55,645 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39216, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-22T03:40:55,646 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2] 2024-11-22T03:40:55,649 DEBUG [Thread-985 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-22T03:40:55,649 DEBUG [Thread-989 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-22T03:40:55,650 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c85114ed5096,42545,1732246786482, seqNum=-1] 2024-11-22T03:40:55,650 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-22T03:40:55,650 DEBUG [Thread-991 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-22T03:40:55,651 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c85114ed5096,42545,1732246786482, seqNum=-1] 2024-11-22T03:40:55,651 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-22T03:40:55,651 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c85114ed5096,42545,1732246786482, seqNum=-1] 2024-11-22T03:40:55,651 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-22T03:40:55,651 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39226, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-22T03:40:55,652 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39228, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-22T03:40:55,653 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39242, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-22T03:40:55,653 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2] 2024-11-22T03:40:55,653 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2] 2024-11-22T03:40:55,656 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2] 2024-11-22T03:40:55,680 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-22T03:40:55,680 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f0af25cc84372b1efb01f18522e2d562 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-22T03:40:55,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(8855): Flush requested on f0af25cc84372b1efb01f18522e2d562 2024-11-22T03:40:55,701 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/.tmp/A/8332a7a1776e4cc894739dace3dcea08 is 50, key is test_row_0/A:col10/1732246855649/Put/seqid=0 2024-11-22T03:40:55,712 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742031_1207 (size=14341) 2024-11-22T03:40:55,720 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:55,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39228 deadline: 1732246915713, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:55,720 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:55,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39154 deadline: 1732246915714, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:55,721 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:55,721 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:55,721 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:40:55,720 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:55,721 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:55,721 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:55,722 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:55,722 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:40:55,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39216 deadline: 1732246915713, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:55,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39242 deadline: 1732246915717, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:55,722 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:55,722 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:55,722 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:40:55,722 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:55,722 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/.tmp/A/8332a7a1776e4cc894739dace3dcea08 2024-11-22T03:40:55,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39226 deadline: 1732246915718, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:55,723 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:55,723 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:55,723 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:40:55,723 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:55,723 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:55,723 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:40:55,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-22T03:40:55,776 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/.tmp/B/0602f6ff1008455e94d81b162d6d8c3b is 50, key is test_row_0/B:col10/1732246855649/Put/seqid=0 2024-11-22T03:40:55,786 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42545 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-22T03:40:55,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562. 2024-11-22T03:40:55,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562. as already flushing 2024-11-22T03:40:55,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562. 2024-11-22T03:40:55,787 ERROR [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => f0af25cc84372b1efb01f18522e2d562, NAME => 'TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:40:55,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => f0af25cc84372b1efb01f18522e2d562, NAME => 'TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:40:55,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42627 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f0af25cc84372b1efb01f18522e2d562, NAME => 'TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f0af25cc84372b1efb01f18522e2d562, NAME => 'TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:40:55,822 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742032_1208 (size=12001) 2024-11-22T03:40:55,830 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:55,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39228 deadline: 1732246915828, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:55,830 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:55,830 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:55,830 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:40:55,831 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:55,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39154 deadline: 1732246915828, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:55,831 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:55,831 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:55,831 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:55,831 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:40:55,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39242 deadline: 1732246915828, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:55,831 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:55,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39216 deadline: 1732246915828, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:55,832 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:55,832 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:55,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39226 deadline: 1732246915829, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:55,832 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:55,832 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:40:55,832 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:55,832 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:55,832 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:40:55,832 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:55,832 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:55,832 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:40:55,940 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42545 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-22T03:40:55,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562. 2024-11-22T03:40:55,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562. as already flushing 2024-11-22T03:40:55,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562. 2024-11-22T03:40:55,940 ERROR [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => f0af25cc84372b1efb01f18522e2d562, NAME => 'TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:40:55,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => f0af25cc84372b1efb01f18522e2d562, NAME => 'TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:40:55,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42627 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f0af25cc84372b1efb01f18522e2d562, NAME => 'TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f0af25cc84372b1efb01f18522e2d562, NAME => 'TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:40:55,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-22T03:40:56,040 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:56,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39228 deadline: 1732246916038, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:56,041 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:56,041 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:56,041 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:56,041 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:40:56,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39154 deadline: 1732246916039, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:56,041 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:56,041 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:56,041 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:40:56,041 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:56,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39216 deadline: 1732246916039, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:56,041 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:56,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39242 deadline: 1732246916039, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:56,042 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:56,042 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:56,042 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:40:56,042 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:56,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39226 deadline: 1732246916039, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:56,042 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:56,042 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:56,042 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:40:56,042 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:56,042 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:56,042 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:40:56,093 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42545 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-22T03:40:56,093 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562. 2024-11-22T03:40:56,093 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562. as already flushing 2024-11-22T03:40:56,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562. 2024-11-22T03:40:56,094 ERROR [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => f0af25cc84372b1efb01f18522e2d562, NAME => 'TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:40:56,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => f0af25cc84372b1efb01f18522e2d562, NAME => 'TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:40:56,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42627 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f0af25cc84372b1efb01f18522e2d562, NAME => 'TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f0af25cc84372b1efb01f18522e2d562, NAME => 'TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:40:56,223 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/.tmp/B/0602f6ff1008455e94d81b162d6d8c3b 2024-11-22T03:40:56,243 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/.tmp/C/d0b829e9f5db4a1bbee3ee88b56800f2 is 50, key is test_row_0/C:col10/1732246855649/Put/seqid=0 2024-11-22T03:40:56,247 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42545 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-22T03:40:56,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562. 2024-11-22T03:40:56,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562. as already flushing 2024-11-22T03:40:56,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562. 2024-11-22T03:40:56,247 ERROR [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => f0af25cc84372b1efb01f18522e2d562, NAME => 'TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:40:56,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => f0af25cc84372b1efb01f18522e2d562, NAME => 'TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:40:56,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42627 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f0af25cc84372b1efb01f18522e2d562, NAME => 'TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f0af25cc84372b1efb01f18522e2d562, NAME => 'TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:40:56,251 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742033_1209 (size=12001) 2024-11-22T03:40:56,251 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/.tmp/C/d0b829e9f5db4a1bbee3ee88b56800f2 2024-11-22T03:40:56,258 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/.tmp/A/8332a7a1776e4cc894739dace3dcea08 as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/A/8332a7a1776e4cc894739dace3dcea08 2024-11-22T03:40:56,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-22T03:40:56,263 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/A/8332a7a1776e4cc894739dace3dcea08, entries=200, sequenceid=13, filesize=14.0 K 2024-11-22T03:40:56,264 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/.tmp/B/0602f6ff1008455e94d81b162d6d8c3b as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/B/0602f6ff1008455e94d81b162d6d8c3b 2024-11-22T03:40:56,268 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/B/0602f6ff1008455e94d81b162d6d8c3b, entries=150, sequenceid=13, filesize=11.7 K 2024-11-22T03:40:56,272 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/.tmp/C/d0b829e9f5db4a1bbee3ee88b56800f2 as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/C/d0b829e9f5db4a1bbee3ee88b56800f2 2024-11-22T03:40:56,276 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/C/d0b829e9f5db4a1bbee3ee88b56800f2, entries=150, sequenceid=13, filesize=11.7 K 2024-11-22T03:40:56,278 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=147.60 KB/151140 for f0af25cc84372b1efb01f18522e2d562 in 597ms, sequenceid=13, compaction requested=false 2024-11-22T03:40:56,278 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f0af25cc84372b1efb01f18522e2d562: 2024-11-22T03:40:56,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(8855): Flush requested on f0af25cc84372b1efb01f18522e2d562 2024-11-22T03:40:56,351 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-22T03:40:56,351 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f0af25cc84372b1efb01f18522e2d562 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-22T03:40:56,357 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:56,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39226 deadline: 1732246916353, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:56,357 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/.tmp/A/360dac24cf294908be14f8507c0a9cef is 50, key is test_row_0/A:col10/1732246856351/Put/seqid=0 2024-11-22T03:40:56,357 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:56,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39228 deadline: 1732246916355, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:56,357 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:56,357 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:56,357 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:40:56,357 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:56,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39216 deadline: 1732246916356, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:56,358 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:56,358 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:56,358 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:56,358 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:40:56,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39154 deadline: 1732246916357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:56,358 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:56,358 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:56,358 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:40:56,358 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:56,358 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:56,358 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:40:56,360 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:56,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39242 deadline: 1732246916358, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:56,361 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:56,361 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:56,361 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:40:56,361 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742034_1210 (size=12001) 2024-11-22T03:40:56,361 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58.14 KB at sequenceid=42 (bloomFilter=true), to=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/.tmp/A/360dac24cf294908be14f8507c0a9cef 2024-11-22T03:40:56,369 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/.tmp/B/8c6ce8a97d054672a56b1e4b6ccfeede is 50, key is test_row_0/B:col10/1732246856351/Put/seqid=0 2024-11-22T03:40:56,375 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742035_1211 (size=12001) 2024-11-22T03:40:56,400 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42545 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-22T03:40:56,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562. 2024-11-22T03:40:56,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562. as already flushing 2024-11-22T03:40:56,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562. 2024-11-22T03:40:56,400 ERROR [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => f0af25cc84372b1efb01f18522e2d562, NAME => 'TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:40:56,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => f0af25cc84372b1efb01f18522e2d562, NAME => 'TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:40:56,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42627 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f0af25cc84372b1efb01f18522e2d562, NAME => 'TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f0af25cc84372b1efb01f18522e2d562, NAME => 'TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:40:56,469 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:56,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39226 deadline: 1732246916468, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:56,470 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:56,470 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:56,470 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:40:56,470 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:56,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39216 deadline: 1732246916468, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:56,470 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:56,470 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:56,470 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:56,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39228 deadline: 1732246916468, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:56,470 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:40:56,470 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:56,470 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:56,470 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:56,470 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:40:56,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39154 deadline: 1732246916468, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:56,471 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:56,471 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:56,471 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:56,471 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:40:56,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39242 deadline: 1732246916468, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:56,471 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:56,471 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:56,471 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:40:56,553 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42545 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-22T03:40:56,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562. 2024-11-22T03:40:56,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562. as already flushing 2024-11-22T03:40:56,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562. 2024-11-22T03:40:56,553 ERROR [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => f0af25cc84372b1efb01f18522e2d562, NAME => 'TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:40:56,554 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => f0af25cc84372b1efb01f18522e2d562, NAME => 'TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:40:56,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42627 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f0af25cc84372b1efb01f18522e2d562, NAME => 'TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f0af25cc84372b1efb01f18522e2d562, NAME => 'TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:40:56,679 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:56,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39226 deadline: 1732246916678, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:56,679 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:56,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39216 deadline: 1732246916678, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:56,679 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:56,679 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:56,680 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:40:56,680 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:56,680 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:56,680 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:40:56,680 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:56,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39228 deadline: 1732246916678, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:56,680 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:56,680 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:56,680 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:56,680 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:40:56,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39154 deadline: 1732246916678, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:56,680 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:56,680 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:56,680 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:56,680 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:40:56,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39242 deadline: 1732246916678, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:56,680 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:56,680 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:56,680 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:40:56,706 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42545 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-22T03:40:56,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562. 2024-11-22T03:40:56,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562. as already flushing 2024-11-22T03:40:56,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562. 2024-11-22T03:40:56,706 ERROR [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => f0af25cc84372b1efb01f18522e2d562, NAME => 'TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:40:56,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => f0af25cc84372b1efb01f18522e2d562, NAME => 'TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:40:56,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42627 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f0af25cc84372b1efb01f18522e2d562, NAME => 'TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f0af25cc84372b1efb01f18522e2d562, NAME => 'TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:40:56,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-22T03:40:56,776 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58.14 KB at sequenceid=42 (bloomFilter=true), to=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/.tmp/B/8c6ce8a97d054672a56b1e4b6ccfeede 2024-11-22T03:40:56,783 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/.tmp/C/c56fbdd13ddf4b9c9ae351299d7311b8 is 50, key is test_row_0/C:col10/1732246856351/Put/seqid=0 2024-11-22T03:40:56,786 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742036_1212 (size=12001) 2024-11-22T03:40:56,859 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42545 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-22T03:40:56,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562. 2024-11-22T03:40:56,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562. as already flushing 2024-11-22T03:40:56,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562. 2024-11-22T03:40:56,859 ERROR [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => f0af25cc84372b1efb01f18522e2d562, NAME => 'TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:40:56,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => f0af25cc84372b1efb01f18522e2d562, NAME => 'TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:40:56,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42627 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f0af25cc84372b1efb01f18522e2d562, NAME => 'TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f0af25cc84372b1efb01f18522e2d562, NAME => 'TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:40:56,990 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:56,990 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:56,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39216 deadline: 1732246916988, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:56,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39226 deadline: 1732246916988, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:56,991 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:56,991 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:56,991 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:56,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39228 deadline: 1732246916988, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:56,991 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:56,991 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:56,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39154 deadline: 1732246916988, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:56,991 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:40:56,991 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:56,991 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:40:56,991 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:56,991 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:56,991 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:56,991 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:56,991 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:56,991 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:40:56,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39242 deadline: 1732246916988, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:56,991 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:40:56,991 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:56,991 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:56,991 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:40:57,012 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42545 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-22T03:40:57,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562. 2024-11-22T03:40:57,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562. as already flushing 2024-11-22T03:40:57,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562. 2024-11-22T03:40:57,012 ERROR [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => f0af25cc84372b1efb01f18522e2d562, NAME => 'TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:40:57,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => f0af25cc84372b1efb01f18522e2d562, NAME => 'TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:40:57,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42627 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f0af25cc84372b1efb01f18522e2d562, NAME => 'TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f0af25cc84372b1efb01f18522e2d562, NAME => 'TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:40:57,164 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42545 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-22T03:40:57,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562. 2024-11-22T03:40:57,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562. as already flushing 2024-11-22T03:40:57,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562. 2024-11-22T03:40:57,165 ERROR [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => f0af25cc84372b1efb01f18522e2d562, NAME => 'TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:40:57,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => f0af25cc84372b1efb01f18522e2d562, NAME => 'TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:40:57,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42627 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f0af25cc84372b1efb01f18522e2d562, NAME => 'TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f0af25cc84372b1efb01f18522e2d562, NAME => 'TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:40:57,187 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58.14 KB at sequenceid=42 (bloomFilter=true), to=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/.tmp/C/c56fbdd13ddf4b9c9ae351299d7311b8 2024-11-22T03:40:57,191 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/.tmp/A/360dac24cf294908be14f8507c0a9cef as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/A/360dac24cf294908be14f8507c0a9cef 2024-11-22T03:40:57,195 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/A/360dac24cf294908be14f8507c0a9cef, entries=150, sequenceid=42, filesize=11.7 K 2024-11-22T03:40:57,195 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/.tmp/B/8c6ce8a97d054672a56b1e4b6ccfeede as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/B/8c6ce8a97d054672a56b1e4b6ccfeede 2024-11-22T03:40:57,200 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/B/8c6ce8a97d054672a56b1e4b6ccfeede, entries=150, sequenceid=42, filesize=11.7 K 2024-11-22T03:40:57,201 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/.tmp/C/c56fbdd13ddf4b9c9ae351299d7311b8 as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/C/c56fbdd13ddf4b9c9ae351299d7311b8 2024-11-22T03:40:57,205 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/C/c56fbdd13ddf4b9c9ae351299d7311b8, entries=150, sequenceid=42, filesize=11.7 K 2024-11-22T03:40:57,206 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~174.43 KB/178620, heapSize ~457.73 KB/468720, currentSize=33.54 KB/34350 for f0af25cc84372b1efb01f18522e2d562 in 855ms, sequenceid=42, compaction requested=false 2024-11-22T03:40:57,206 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f0af25cc84372b1efb01f18522e2d562: 2024-11-22T03:40:57,317 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42545 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-22T03:40:57,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562. 2024-11-22T03:40:57,318 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2902): Flushing f0af25cc84372b1efb01f18522e2d562 3/3 column families, dataSize=33.54 KB heapSize=88.64 KB 2024-11-22T03:40:57,321 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/.tmp/A/ce9c130885994d718d8cded9e821076d is 50, key is test_row_0/A:col10/1732246856356/Put/seqid=0 2024-11-22T03:40:57,328 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742037_1213 (size=12001) 2024-11-22T03:40:57,329 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/.tmp/A/ce9c130885994d718d8cded9e821076d 2024-11-22T03:40:57,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/.tmp/B/597b6c5cb45c4396a1ed47468cc82c9b is 50, key is test_row_0/B:col10/1732246856356/Put/seqid=0 2024-11-22T03:40:57,341 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742038_1214 (size=12001) 2024-11-22T03:40:57,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(8855): Flush requested on f0af25cc84372b1efb01f18522e2d562 2024-11-22T03:40:57,503 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562. as already flushing 2024-11-22T03:40:57,526 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:57,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39216 deadline: 1732246917523, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:57,527 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:57,527 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:57,527 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:40:57,528 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:57,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39154 deadline: 1732246917524, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:57,528 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:57,528 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:57,528 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:57,529 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:40:57,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39242 deadline: 1732246917525, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:57,529 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:57,529 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:57,529 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:40:57,530 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:57,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39228 deadline: 1732246917527, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:57,531 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:57,531 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:57,531 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:57,531 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:40:57,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39226 deadline: 1732246917527, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:57,532 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:57,532 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:57,532 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:40:57,629 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:57,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39216 deadline: 1732246917629, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:57,630 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:57,630 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:57,630 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:40:57,641 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:57,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39154 deadline: 1732246917639, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:57,641 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:57,641 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:57,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39242 deadline: 1732246917639, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:57,641 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:57,641 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:40:57,641 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:57,641 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:57,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39228 deadline: 1732246917639, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:57,641 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:57,641 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:40:57,642 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:57,642 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:57,642 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:57,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39226 deadline: 1732246917639, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:57,642 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:40:57,642 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:57,642 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:57,642 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:40:57,742 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/.tmp/B/597b6c5cb45c4396a1ed47468cc82c9b 2024-11-22T03:40:57,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/.tmp/C/5df9af5387404f3888e0cd48ee9e075d is 50, key is test_row_0/C:col10/1732246856356/Put/seqid=0 2024-11-22T03:40:57,758 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742039_1215 (size=12001) 2024-11-22T03:40:57,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-22T03:40:57,840 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:57,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39216 deadline: 1732246917839, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:57,840 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:57,840 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:57,840 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:40:57,849 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:57,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39154 deadline: 1732246917848, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:57,850 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:57,850 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:57,850 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:57,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39242 deadline: 1732246917848, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:57,850 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:40:57,850 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:57,850 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:57,850 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:57,850 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:40:57,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39226 deadline: 1732246917848, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:57,850 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:57,850 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:57,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39228 deadline: 1732246917849, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:57,851 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:57,851 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:40:57,851 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:57,851 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:57,851 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:40:58,152 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:58,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39216 deadline: 1732246918149, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:58,152 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:58,153 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:58,153 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:40:58,159 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/.tmp/C/5df9af5387404f3888e0cd48ee9e075d 2024-11-22T03:40:58,161 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:58,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39154 deadline: 1732246918159, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:58,162 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:58,162 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:58,162 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:58,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39242 deadline: 1732246918159, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:58,162 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:40:58,162 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:58,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39226 deadline: 1732246918159, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:58,162 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:58,162 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:58,162 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:40:58,162 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:58,162 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:58,162 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:40:58,162 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:58,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39228 deadline: 1732246918159, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:58,163 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:58,163 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:58,163 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:40:58,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/.tmp/A/ce9c130885994d718d8cded9e821076d as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/A/ce9c130885994d718d8cded9e821076d 2024-11-22T03:40:58,167 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/A/ce9c130885994d718d8cded9e821076d, entries=150, sequenceid=50, filesize=11.7 K 2024-11-22T03:40:58,168 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/.tmp/B/597b6c5cb45c4396a1ed47468cc82c9b as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/B/597b6c5cb45c4396a1ed47468cc82c9b 2024-11-22T03:40:58,172 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/B/597b6c5cb45c4396a1ed47468cc82c9b, entries=150, sequenceid=50, filesize=11.7 K 2024-11-22T03:40:58,173 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/.tmp/C/5df9af5387404f3888e0cd48ee9e075d as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/C/5df9af5387404f3888e0cd48ee9e075d 2024-11-22T03:40:58,177 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/C/5df9af5387404f3888e0cd48ee9e075d, entries=150, sequenceid=50, filesize=11.7 K 2024-11-22T03:40:58,178 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=167.72 KB/171750 for f0af25cc84372b1efb01f18522e2d562 in 861ms, sequenceid=50, compaction requested=true 2024-11-22T03:40:58,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2603): Flush status journal for f0af25cc84372b1efb01f18522e2d562: 2024-11-22T03:40:58,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562. 2024-11-22T03:40:58,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=63 2024-11-22T03:40:58,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42627 {}] master.HMaster(4169): Remote procedure done, pid=63 2024-11-22T03:40:58,181 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=63, resume processing ppid=62 2024-11-22T03:40:58,181 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=63, ppid=62, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.5490 sec 2024-11-22T03:40:58,183 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=62, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees in 2.5590 sec 2024-11-22T03:40:58,573 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-22T03:40:58,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(8855): Flush requested on f0af25cc84372b1efb01f18522e2d562 2024-11-22T03:40:58,661 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-22T03:40:58,661 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f0af25cc84372b1efb01f18522e2d562 3/3 column families, dataSize=174.43 KB heapSize=457.78 KB 2024-11-22T03:40:58,665 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/.tmp/A/5895b4596238410481d41ef161ba469a is 50, key is test_row_0/A:col10/1732246858660/Put/seqid=0 2024-11-22T03:40:58,669 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742040_1216 (size=14341) 2024-11-22T03:40:58,669 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58.14 KB at sequenceid=79 (bloomFilter=true), to=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/.tmp/A/5895b4596238410481d41ef161ba469a 2024-11-22T03:40:58,673 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:58,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39226 deadline: 1732246918669, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:58,673 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:58,673 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:58,673 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:58,673 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:40:58,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39216 deadline: 1732246918670, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:58,673 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:58,673 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:58,674 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:40:58,675 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:58,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39154 deadline: 1732246918671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:58,676 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:58,676 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:58,676 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:58,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39242 deadline: 1732246918672, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:58,676 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:40:58,676 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/.tmp/B/df06dbea9aad4262aa4376083616dfdf is 50, key is test_row_0/B:col10/1732246858660/Put/seqid=0 2024-11-22T03:40:58,676 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:58,676 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:58,676 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:40:58,676 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:58,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39228 deadline: 1732246918673, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:58,677 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:58,677 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:58,677 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:40:58,705 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742041_1217 (size=12001) 2024-11-22T03:40:58,714 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58.14 KB at sequenceid=79 (bloomFilter=true), to=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/.tmp/B/df06dbea9aad4262aa4376083616dfdf 2024-11-22T03:40:58,721 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/.tmp/C/1d0ab2cfdb32417f938379e8d8fbc81f is 50, key is test_row_0/C:col10/1732246858660/Put/seqid=0 2024-11-22T03:40:58,724 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742042_1218 (size=12001) 2024-11-22T03:40:58,725 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58.14 KB at sequenceid=79 (bloomFilter=true), to=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/.tmp/C/1d0ab2cfdb32417f938379e8d8fbc81f 2024-11-22T03:40:58,730 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/.tmp/A/5895b4596238410481d41ef161ba469a as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/A/5895b4596238410481d41ef161ba469a 2024-11-22T03:40:58,739 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/A/5895b4596238410481d41ef161ba469a, entries=200, sequenceid=79, filesize=14.0 K 2024-11-22T03:40:58,740 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/.tmp/B/df06dbea9aad4262aa4376083616dfdf as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/B/df06dbea9aad4262aa4376083616dfdf 2024-11-22T03:40:58,745 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/B/df06dbea9aad4262aa4376083616dfdf, entries=150, sequenceid=79, filesize=11.7 K 2024-11-22T03:40:58,746 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/.tmp/C/1d0ab2cfdb32417f938379e8d8fbc81f as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/C/1d0ab2cfdb32417f938379e8d8fbc81f 2024-11-22T03:40:58,750 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/C/1d0ab2cfdb32417f938379e8d8fbc81f, entries=150, sequenceid=79, filesize=11.7 K 2024-11-22T03:40:58,751 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~174.43 KB/178620, heapSize ~457.73 KB/468720, currentSize=33.54 KB/34350 for f0af25cc84372b1efb01f18522e2d562 in 89ms, sequenceid=79, compaction requested=true 2024-11-22T03:40:58,751 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f0af25cc84372b1efb01f18522e2d562: 2024-11-22T03:40:58,751 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f0af25cc84372b1efb01f18522e2d562:A, priority=-2147483648, current under compaction store size is 1 2024-11-22T03:40:58,751 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T03:40:58,751 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-22T03:40:58,751 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f0af25cc84372b1efb01f18522e2d562:B, priority=-2147483648, current under compaction store size is 2 2024-11-22T03:40:58,751 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T03:40:58,751 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-22T03:40:58,751 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f0af25cc84372b1efb01f18522e2d562:C, priority=-2147483648, current under compaction store size is 3 2024-11-22T03:40:58,751 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T03:40:58,752 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 52684 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-22T03:40:58,752 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-22T03:40:58,752 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.HStore(1541): f0af25cc84372b1efb01f18522e2d562/A is initiating minor compaction (all files) 2024-11-22T03:40:58,752 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HStore(1541): f0af25cc84372b1efb01f18522e2d562/B is initiating minor compaction (all files) 2024-11-22T03:40:58,752 INFO [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f0af25cc84372b1efb01f18522e2d562/B in TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562. 2024-11-22T03:40:58,752 INFO [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f0af25cc84372b1efb01f18522e2d562/A in TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562. 2024-11-22T03:40:58,752 INFO [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/B/0602f6ff1008455e94d81b162d6d8c3b, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/B/8c6ce8a97d054672a56b1e4b6ccfeede, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/B/597b6c5cb45c4396a1ed47468cc82c9b, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/B/df06dbea9aad4262aa4376083616dfdf] into tmpdir=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/.tmp, totalSize=46.9 K 2024-11-22T03:40:58,752 INFO [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/A/8332a7a1776e4cc894739dace3dcea08, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/A/360dac24cf294908be14f8507c0a9cef, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/A/ce9c130885994d718d8cded9e821076d, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/A/5895b4596238410481d41ef161ba469a] into tmpdir=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/.tmp, totalSize=51.4 K 2024-11-22T03:40:58,753 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8332a7a1776e4cc894739dace3dcea08, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1732246855636 2024-11-22T03:40:58,753 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] compactions.Compactor(225): Compacting 0602f6ff1008455e94d81b162d6d8c3b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1732246855636 2024-11-22T03:40:58,755 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] compactions.Compactor(225): Compacting 8c6ce8a97d054672a56b1e4b6ccfeede, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=42, earliestPutTs=1732246856351 2024-11-22T03:40:58,755 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] compactions.Compactor(225): Compacting 360dac24cf294908be14f8507c0a9cef, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=42, earliestPutTs=1732246856351 2024-11-22T03:40:58,755 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] compactions.Compactor(225): Compacting ce9c130885994d718d8cded9e821076d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1732246856354 2024-11-22T03:40:58,756 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] compactions.Compactor(225): Compacting 597b6c5cb45c4396a1ed47468cc82c9b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1732246856354 2024-11-22T03:40:58,756 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5895b4596238410481d41ef161ba469a, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=79, earliestPutTs=1732246857517 2024-11-22T03:40:58,756 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] compactions.Compactor(225): Compacting df06dbea9aad4262aa4376083616dfdf, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=79, earliestPutTs=1732246857517 2024-11-22T03:40:58,771 INFO [RS:0;c85114ed5096:42545-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f0af25cc84372b1efb01f18522e2d562#A#compaction#183 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-22T03:40:58,772 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/.tmp/A/0313ed4a34164ca48471ecb993831253 is 50, key is test_row_0/A:col10/1732246858660/Put/seqid=0 2024-11-22T03:40:58,779 INFO [RS:0;c85114ed5096:42545-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f0af25cc84372b1efb01f18522e2d562#B#compaction#184 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-22T03:40:58,780 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/.tmp/B/3eca758af85c41278c085e6a74467150 is 50, key is test_row_0/B:col10/1732246858660/Put/seqid=0 2024-11-22T03:40:58,781 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-22T03:40:58,782 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f0af25cc84372b1efb01f18522e2d562 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-22T03:40:58,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(8855): Flush requested on f0af25cc84372b1efb01f18522e2d562 2024-11-22T03:40:58,814 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/.tmp/A/8553f5e1fd924c6cb0b35ce63adb703f is 50, key is test_row_0/A:col10/1732246858780/Put/seqid=0 2024-11-22T03:40:58,833 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:58,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39154 deadline: 1732246918825, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:58,834 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:58,834 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:58,834 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:40:58,834 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:58,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39228 deadline: 1732246918826, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:58,834 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:58,834 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:58,834 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:40:58,835 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:58,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39242 deadline: 1732246918828, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:58,835 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:58,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39216 deadline: 1732246918829, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:58,836 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:58,836 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:58,836 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:40:58,836 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:58,836 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:58,836 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:40:58,837 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742043_1219 (size=12139) 2024-11-22T03:40:58,844 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/.tmp/A/0313ed4a34164ca48471ecb993831253 as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/A/0313ed4a34164ca48471ecb993831253 2024-11-22T03:40:58,858 INFO [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in f0af25cc84372b1efb01f18522e2d562/A of f0af25cc84372b1efb01f18522e2d562 into 0313ed4a34164ca48471ecb993831253(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T03:40:58,858 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f0af25cc84372b1efb01f18522e2d562: 2024-11-22T03:40:58,858 INFO [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., storeName=f0af25cc84372b1efb01f18522e2d562/A, priority=12, startTime=1732246858751; duration=0sec 2024-11-22T03:40:58,858 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T03:40:58,858 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f0af25cc84372b1efb01f18522e2d562:A 2024-11-22T03:40:58,858 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-22T03:40:58,860 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-22T03:40:58,860 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.HStore(1541): f0af25cc84372b1efb01f18522e2d562/C is initiating minor compaction (all files) 2024-11-22T03:40:58,860 INFO [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f0af25cc84372b1efb01f18522e2d562/C in TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562. 2024-11-22T03:40:58,861 INFO [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/C/d0b829e9f5db4a1bbee3ee88b56800f2, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/C/c56fbdd13ddf4b9c9ae351299d7311b8, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/C/5df9af5387404f3888e0cd48ee9e075d, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/C/1d0ab2cfdb32417f938379e8d8fbc81f] into tmpdir=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/.tmp, totalSize=46.9 K 2024-11-22T03:40:58,861 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] compactions.Compactor(225): Compacting d0b829e9f5db4a1bbee3ee88b56800f2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1732246855636 2024-11-22T03:40:58,862 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] compactions.Compactor(225): Compacting c56fbdd13ddf4b9c9ae351299d7311b8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=42, earliestPutTs=1732246856351 2024-11-22T03:40:58,862 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5df9af5387404f3888e0cd48ee9e075d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1732246856354 2024-11-22T03:40:58,862 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1d0ab2cfdb32417f938379e8d8fbc81f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=79, earliestPutTs=1732246857517 2024-11-22T03:40:58,868 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742044_1220 (size=12139) 2024-11-22T03:40:58,883 INFO [RS:0;c85114ed5096:42545-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f0af25cc84372b1efb01f18522e2d562#C#compaction#186 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-22T03:40:58,884 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/.tmp/C/444c524511c246a193f089a5386fbc20 is 50, key is test_row_0/C:col10/1732246858660/Put/seqid=0 2024-11-22T03:40:58,891 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742045_1221 (size=14341) 2024-11-22T03:40:58,891 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=91 (bloomFilter=true), to=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/.tmp/A/8553f5e1fd924c6cb0b35ce63adb703f 2024-11-22T03:40:58,898 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/.tmp/B/fb8942f6a96d4d49999f5324477d3a6e is 50, key is test_row_0/B:col10/1732246858780/Put/seqid=0 2024-11-22T03:40:58,901 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742046_1222 (size=12139) 2024-11-22T03:40:58,906 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742047_1223 (size=12001) 2024-11-22T03:40:58,940 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:58,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39154 deadline: 1732246918939, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:58,940 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:58,940 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:58,941 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:58,941 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:40:58,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39228 deadline: 1732246918939, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:58,941 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:58,941 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:58,941 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:40:58,941 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:58,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39216 deadline: 1732246918939, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:58,942 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:58,942 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:58,942 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:58,942 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:40:58,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39242 deadline: 1732246918939, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:58,942 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:58,942 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:58,942 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:40:59,151 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:59,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39154 deadline: 1732246919148, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:59,151 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:59,151 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:59,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39228 deadline: 1732246919149, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:59,152 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:59,152 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:40:59,152 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:59,152 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:59,152 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:59,152 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:40:59,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39216 deadline: 1732246919149, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:59,152 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:59,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39242 deadline: 1732246919149, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:59,152 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:59,152 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:59,152 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:40:59,152 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:59,152 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:59,152 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:40:59,274 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/.tmp/B/3eca758af85c41278c085e6a74467150 as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/B/3eca758af85c41278c085e6a74467150 2024-11-22T03:40:59,278 INFO [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in f0af25cc84372b1efb01f18522e2d562/B of f0af25cc84372b1efb01f18522e2d562 into 3eca758af85c41278c085e6a74467150(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T03:40:59,278 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f0af25cc84372b1efb01f18522e2d562: 2024-11-22T03:40:59,279 INFO [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., storeName=f0af25cc84372b1efb01f18522e2d562/B, priority=12, startTime=1732246858751; duration=0sec 2024-11-22T03:40:59,279 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T03:40:59,279 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f0af25cc84372b1efb01f18522e2d562:B 2024-11-22T03:40:59,306 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/.tmp/C/444c524511c246a193f089a5386fbc20 as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/C/444c524511c246a193f089a5386fbc20 2024-11-22T03:40:59,307 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=91 (bloomFilter=true), to=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/.tmp/B/fb8942f6a96d4d49999f5324477d3a6e 2024-11-22T03:40:59,312 INFO [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in f0af25cc84372b1efb01f18522e2d562/C of f0af25cc84372b1efb01f18522e2d562 into 444c524511c246a193f089a5386fbc20(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T03:40:59,312 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f0af25cc84372b1efb01f18522e2d562: 2024-11-22T03:40:59,312 INFO [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., storeName=f0af25cc84372b1efb01f18522e2d562/C, priority=12, startTime=1732246858751; duration=0sec 2024-11-22T03:40:59,312 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T03:40:59,312 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f0af25cc84372b1efb01f18522e2d562:C 2024-11-22T03:40:59,322 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/.tmp/C/9a5b3dbcec1f45358f56f8b9a360a457 is 50, key is test_row_0/C:col10/1732246858780/Put/seqid=0 2024-11-22T03:40:59,326 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742048_1224 (size=12001) 2024-11-22T03:40:59,460 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:59,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39154 deadline: 1732246919458, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:59,460 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:59,460 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:59,460 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:59,460 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:40:59,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39228 deadline: 1732246919458, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:59,461 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:59,461 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:59,461 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:59,461 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:40:59,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39216 deadline: 1732246919458, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:59,461 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:59,461 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:59,461 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:40:59,461 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:59,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39242 deadline: 1732246919458, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:59,461 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:59,462 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:59,462 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:40:59,680 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:40:59,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39226 deadline: 1732246919678, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 2024-11-22T03:40:59,680 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to c85114ed5096,42545,1732246786482 for 'test_row_2' in f0af25cc84372b1efb01f18522e2d562 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2153 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-22T03:40:59,681 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:59,681 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:40:59,681 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:40:59,727 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=91 (bloomFilter=true), to=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/.tmp/C/9a5b3dbcec1f45358f56f8b9a360a457 2024-11-22T03:40:59,731 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/.tmp/A/8553f5e1fd924c6cb0b35ce63adb703f as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/A/8553f5e1fd924c6cb0b35ce63adb703f 2024-11-22T03:40:59,735 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/A/8553f5e1fd924c6cb0b35ce63adb703f, entries=200, sequenceid=91, filesize=14.0 K 2024-11-22T03:40:59,736 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/.tmp/B/fb8942f6a96d4d49999f5324477d3a6e as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/B/fb8942f6a96d4d49999f5324477d3a6e 2024-11-22T03:40:59,740 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/B/fb8942f6a96d4d49999f5324477d3a6e, entries=150, sequenceid=91, filesize=11.7 K 2024-11-22T03:40:59,741 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/.tmp/C/9a5b3dbcec1f45358f56f8b9a360a457 as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/C/9a5b3dbcec1f45358f56f8b9a360a457 2024-11-22T03:40:59,745 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/C/9a5b3dbcec1f45358f56f8b9a360a457, entries=150, sequenceid=91, filesize=11.7 K 2024-11-22T03:40:59,746 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for f0af25cc84372b1efb01f18522e2d562 in 964ms, sequenceid=91, compaction requested=false 2024-11-22T03:40:59,746 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f0af25cc84372b1efb01f18522e2d562: 2024-11-22T03:40:59,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-22T03:40:59,790 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-22T03:40:59,790 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42627 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-22T03:40:59,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42627 {}] procedure2.ProcedureExecutor(1139): Stored pid=64, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees 2024-11-22T03:40:59,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-22T03:40:59,792 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=64, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-22T03:40:59,792 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=64, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-22T03:40:59,792 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=65, ppid=64, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-22T03:40:59,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-22T03:40:59,945 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42545 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-22T03:40:59,945 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562. 2024-11-22T03:40:59,945 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2902): Flushing f0af25cc84372b1efb01f18522e2d562 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-22T03:40:59,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/.tmp/A/544b39b880cc4720b45461244ceeebb1 is 50, key is test_row_0/A:col10/1732246858827/Put/seqid=0 2024-11-22T03:40:59,953 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742049_1225 (size=12001) 2024-11-22T03:40:59,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(8855): Flush requested on f0af25cc84372b1efb01f18522e2d562 2024-11-22T03:40:59,999 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562. as already flushing 2024-11-22T03:41:00,009 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:00,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39228 deadline: 1732246920004, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:00,009 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:00,009 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:00,009 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:00,009 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:41:00,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39216 deadline: 1732246920004, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:00,009 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:00,009 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:00,009 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:41:00,009 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:00,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39242 deadline: 1732246920004, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:00,010 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:00,010 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:00,010 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:41:00,012 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:00,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39154 deadline: 1732246920009, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:00,013 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:00,013 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:00,013 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:41:00,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-22T03:41:00,121 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:00,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39228 deadline: 1732246920118, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:00,121 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:00,121 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:00,121 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:00,122 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:41:00,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39216 deadline: 1732246920118, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:00,122 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:00,122 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:00,122 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:41:00,122 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:00,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39242 deadline: 1732246920118, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:00,122 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:00,122 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:00,122 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:41:00,122 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:00,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39154 deadline: 1732246920119, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:00,122 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:00,122 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:00,122 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:41:00,330 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:00,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39228 deadline: 1732246920328, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:00,330 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:00,330 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:00,330 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:00,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39216 deadline: 1732246920328, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:00,330 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:41:00,330 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:00,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39242 deadline: 1732246920328, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:00,330 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:00,330 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:00,330 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:41:00,330 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:00,330 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:00,330 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:00,330 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:41:00,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39154 deadline: 1732246920328, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:00,331 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:00,331 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:00,331 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:41:00,354 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=118 (bloomFilter=true), to=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/.tmp/A/544b39b880cc4720b45461244ceeebb1 2024-11-22T03:41:00,362 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/.tmp/B/3e4065072da944e3afdec445fc6f29e8 is 50, key is test_row_0/B:col10/1732246858827/Put/seqid=0 2024-11-22T03:41:00,374 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742050_1226 (size=12001) 2024-11-22T03:41:00,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-22T03:41:00,642 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:00,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39228 deadline: 1732246920639, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:00,642 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:00,642 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:00,642 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:00,642 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:41:00,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39216 deadline: 1732246920639, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:00,642 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:00,642 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:00,642 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:00,642 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:41:00,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39242 deadline: 1732246920639, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:00,643 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:00,643 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:00,643 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:00,643 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:41:00,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39154 deadline: 1732246920639, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:00,643 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:00,643 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:00,643 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:41:00,775 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=118 (bloomFilter=true), to=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/.tmp/B/3e4065072da944e3afdec445fc6f29e8 2024-11-22T03:41:00,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/.tmp/C/958da13a301a4354aa1612841c3ce7d4 is 50, key is test_row_0/C:col10/1732246858827/Put/seqid=0 2024-11-22T03:41:00,790 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742051_1227 (size=12001) 2024-11-22T03:41:00,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-22T03:41:01,149 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:01,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39228 deadline: 1732246921148, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:01,150 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:01,150 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:01,150 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:01,150 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:41:01,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39216 deadline: 1732246921148, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:01,150 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:01,150 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:01,150 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:41:01,150 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:01,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39154 deadline: 1732246921148, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:01,150 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:01,150 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:01,150 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:41:01,150 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:01,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39242 deadline: 1732246921148, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:01,151 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:01,151 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:01,151 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:41:01,191 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=118 (bloomFilter=true), to=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/.tmp/C/958da13a301a4354aa1612841c3ce7d4 2024-11-22T03:41:01,195 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/.tmp/A/544b39b880cc4720b45461244ceeebb1 as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/A/544b39b880cc4720b45461244ceeebb1 2024-11-22T03:41:01,198 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/A/544b39b880cc4720b45461244ceeebb1, entries=150, sequenceid=118, filesize=11.7 K 2024-11-22T03:41:01,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/.tmp/B/3e4065072da944e3afdec445fc6f29e8 as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/B/3e4065072da944e3afdec445fc6f29e8 2024-11-22T03:41:01,203 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/B/3e4065072da944e3afdec445fc6f29e8, entries=150, sequenceid=118, filesize=11.7 K 2024-11-22T03:41:01,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/.tmp/C/958da13a301a4354aa1612841c3ce7d4 as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/C/958da13a301a4354aa1612841c3ce7d4 2024-11-22T03:41:01,208 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/C/958da13a301a4354aa1612841c3ce7d4, entries=150, sequenceid=118, filesize=11.7 K 2024-11-22T03:41:01,209 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for f0af25cc84372b1efb01f18522e2d562 in 1263ms, sequenceid=118, compaction requested=true 2024-11-22T03:41:01,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2603): Flush status journal for f0af25cc84372b1efb01f18522e2d562: 2024-11-22T03:41:01,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562. 2024-11-22T03:41:01,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=65 2024-11-22T03:41:01,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42627 {}] master.HMaster(4169): Remote procedure done, pid=65 2024-11-22T03:41:01,212 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=65, resume processing ppid=64 2024-11-22T03:41:01,212 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=65, ppid=64, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4180 sec 2024-11-22T03:41:01,213 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=64, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees in 1.4220 sec 2024-11-22T03:41:01,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(8855): Flush requested on f0af25cc84372b1efb01f18522e2d562 2024-11-22T03:41:01,690 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-22T03:41:01,690 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f0af25cc84372b1efb01f18522e2d562 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-22T03:41:01,694 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/.tmp/A/8795c14b3477408a86ea2056dd8149a5 is 50, key is test_row_0/A:col10/1732246860003/Put/seqid=0 2024-11-22T03:41:01,698 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742052_1228 (size=14441) 2024-11-22T03:41:01,703 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=131 (bloomFilter=true), to=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/.tmp/A/8795c14b3477408a86ea2056dd8149a5 2024-11-22T03:41:01,711 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/.tmp/B/5b09921f55c84d6fa1635753b0e06309 is 50, key is test_row_0/B:col10/1732246860003/Put/seqid=0 2024-11-22T03:41:01,719 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742053_1229 (size=12101) 2024-11-22T03:41:01,780 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:01,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39226 deadline: 1732246921777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:01,780 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:01,780 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:01,780 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:41:01,892 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:01,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39226 deadline: 1732246921889, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:01,892 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:01,892 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:01,892 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:41:01,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-22T03:41:01,939 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-22T03:41:01,939 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42627 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-22T03:41:01,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42627 {}] procedure2.ProcedureExecutor(1139): Stored pid=66, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees 2024-11-22T03:41:01,942 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=66, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-22T03:41:01,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-22T03:41:01,943 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=66, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-22T03:41:01,943 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=67, ppid=66, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-22T03:41:02,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-22T03:41:02,096 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42545 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-22T03:41:02,096 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562. 2024-11-22T03:41:02,096 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562. as already flushing 2024-11-22T03:41:02,097 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562. 2024-11-22T03:41:02,097 ERROR [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => f0af25cc84372b1efb01f18522e2d562, NAME => 'TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:41:02,097 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => f0af25cc84372b1efb01f18522e2d562, NAME => 'TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:41:02,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42627 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f0af25cc84372b1efb01f18522e2d562, NAME => 'TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f0af25cc84372b1efb01f18522e2d562, NAME => 'TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:41:02,101 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:02,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39226 deadline: 1732246922099, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:02,101 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:02,101 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:02,102 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:41:02,119 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=131 (bloomFilter=true), to=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/.tmp/B/5b09921f55c84d6fa1635753b0e06309 2024-11-22T03:41:02,126 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/.tmp/C/04f21fdb166b404380b6ef47531afe11 is 50, key is test_row_0/C:col10/1732246860003/Put/seqid=0 2024-11-22T03:41:02,129 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742054_1230 (size=12101) 2024-11-22T03:41:02,160 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:02,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39228 deadline: 1732246922158, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:02,161 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:02,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39216 deadline: 1732246922158, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:02,161 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:02,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39154 deadline: 1732246922158, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:02,161 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to c85114ed5096,42545,1732246786482 for 'test_row_0' in f0af25cc84372b1efb01f18522e2d562 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2157 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-22T03:41:02,161 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to c85114ed5096,42545,1732246786482 for 'test_row_1' in f0af25cc84372b1efb01f18522e2d562 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2157 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-22T03:41:02,162 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:02,162 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:02,162 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:02,162 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:02,162 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:41:02,162 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:41:02,162 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to c85114ed5096,42545,1732246786482 for 'test_row_1' in f0af25cc84372b1efb01f18522e2d562 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2152 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-22T03:41:02,162 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:02,162 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:02,162 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:41:02,170 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:02,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39242 deadline: 1732246922169, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:02,171 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to c85114ed5096,42545,1732246786482 for 'test_row_1' in f0af25cc84372b1efb01f18522e2d562 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2166 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-22T03:41:02,171 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:02,171 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:02,171 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:41:02,249 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42545 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-22T03:41:02,249 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562. 2024-11-22T03:41:02,249 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562. as already flushing 2024-11-22T03:41:02,249 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562. 2024-11-22T03:41:02,249 ERROR [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => f0af25cc84372b1efb01f18522e2d562, NAME => 'TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:41:02,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => f0af25cc84372b1efb01f18522e2d562, NAME => 'TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:41:02,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42627 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f0af25cc84372b1efb01f18522e2d562, NAME => 'TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f0af25cc84372b1efb01f18522e2d562, NAME => 'TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:41:02,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-22T03:41:02,402 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42545 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-22T03:41:02,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562. 2024-11-22T03:41:02,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562. as already flushing 2024-11-22T03:41:02,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562. 2024-11-22T03:41:02,402 ERROR [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => f0af25cc84372b1efb01f18522e2d562, NAME => 'TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:41:02,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => f0af25cc84372b1efb01f18522e2d562, NAME => 'TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:41:02,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42627 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f0af25cc84372b1efb01f18522e2d562, NAME => 'TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f0af25cc84372b1efb01f18522e2d562, NAME => 'TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:41:02,410 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:02,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39226 deadline: 1732246922409, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:02,410 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:02,410 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:02,410 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:41:02,529 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=131 (bloomFilter=true), to=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/.tmp/C/04f21fdb166b404380b6ef47531afe11 2024-11-22T03:41:02,533 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/.tmp/A/8795c14b3477408a86ea2056dd8149a5 as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/A/8795c14b3477408a86ea2056dd8149a5 2024-11-22T03:41:02,537 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/A/8795c14b3477408a86ea2056dd8149a5, entries=200, sequenceid=131, filesize=14.1 K 2024-11-22T03:41:02,538 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/.tmp/B/5b09921f55c84d6fa1635753b0e06309 as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/B/5b09921f55c84d6fa1635753b0e06309 2024-11-22T03:41:02,545 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/B/5b09921f55c84d6fa1635753b0e06309, entries=150, sequenceid=131, filesize=11.8 K 2024-11-22T03:41:02,546 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/.tmp/C/04f21fdb166b404380b6ef47531afe11 as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/C/04f21fdb166b404380b6ef47531afe11 2024-11-22T03:41:02,551 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/C/04f21fdb166b404380b6ef47531afe11, entries=150, sequenceid=131, filesize=11.8 K 2024-11-22T03:41:02,552 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for f0af25cc84372b1efb01f18522e2d562 in 862ms, sequenceid=131, compaction requested=true 2024-11-22T03:41:02,552 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f0af25cc84372b1efb01f18522e2d562: 2024-11-22T03:41:02,552 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f0af25cc84372b1efb01f18522e2d562:A, priority=-2147483648, current under compaction store size is 1 2024-11-22T03:41:02,552 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T03:41:02,552 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-22T03:41:02,552 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f0af25cc84372b1efb01f18522e2d562:B, priority=-2147483648, current under compaction store size is 2 2024-11-22T03:41:02,552 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T03:41:02,552 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-22T03:41:02,552 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f0af25cc84372b1efb01f18522e2d562:C, priority=-2147483648, current under compaction store size is 3 2024-11-22T03:41:02,552 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T03:41:02,553 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48242 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-22T03:41:02,553 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HStore(1541): f0af25cc84372b1efb01f18522e2d562/B is initiating minor compaction (all files) 2024-11-22T03:41:02,554 INFO [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f0af25cc84372b1efb01f18522e2d562/B in TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562. 2024-11-22T03:41:02,554 INFO [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/B/3eca758af85c41278c085e6a74467150, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/B/fb8942f6a96d4d49999f5324477d3a6e, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/B/3e4065072da944e3afdec445fc6f29e8, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/B/5b09921f55c84d6fa1635753b0e06309] into tmpdir=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/.tmp, totalSize=47.1 K 2024-11-22T03:41:02,554 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 52922 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-22T03:41:02,554 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.HStore(1541): f0af25cc84372b1efb01f18522e2d562/A is initiating minor compaction (all files) 2024-11-22T03:41:02,554 INFO [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f0af25cc84372b1efb01f18522e2d562/A in TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562. 2024-11-22T03:41:02,554 INFO [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/A/0313ed4a34164ca48471ecb993831253, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/A/8553f5e1fd924c6cb0b35ce63adb703f, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/A/544b39b880cc4720b45461244ceeebb1, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/A/8795c14b3477408a86ea2056dd8149a5] into tmpdir=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/.tmp, totalSize=51.7 K 2024-11-22T03:41:02,554 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] compactions.Compactor(225): Compacting 3eca758af85c41278c085e6a74467150, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=79, earliestPutTs=1732246857517 2024-11-22T03:41:02,554 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0313ed4a34164ca48471ecb993831253, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=79, earliestPutTs=1732246857517 2024-11-22T03:41:02,555 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] compactions.Compactor(225): Compacting fb8942f6a96d4d49999f5324477d3a6e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=91, earliestPutTs=1732246858780 2024-11-22T03:41:02,555 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] compactions.Compactor(225): Compacting 3e4065072da944e3afdec445fc6f29e8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1732246858805 2024-11-22T03:41:02,555 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8553f5e1fd924c6cb0b35ce63adb703f, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=91, earliestPutTs=1732246858664 2024-11-22T03:41:02,555 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42545 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-22T03:41:02,555 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562. 2024-11-22T03:41:02,555 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2902): Flushing f0af25cc84372b1efb01f18522e2d562 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-22T03:41:02,555 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] compactions.Compactor(225): Compacting 5b09921f55c84d6fa1635753b0e06309, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1732246860003 2024-11-22T03:41:02,556 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] compactions.Compactor(225): Compacting 544b39b880cc4720b45461244ceeebb1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1732246858805 2024-11-22T03:41:02,556 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8795c14b3477408a86ea2056dd8149a5, keycount=200, bloomtype=ROW, size=14.1 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1732246859972 2024-11-22T03:41:02,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/.tmp/A/7a4710f466d5403a84c7cedf0f23df80 is 50, key is test_row_0/A:col10/1732246861752/Put/seqid=0 2024-11-22T03:41:02,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-22T03:41:02,575 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742055_1231 (size=12151) 2024-11-22T03:41:02,578 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=154 (bloomFilter=true), to=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/.tmp/A/7a4710f466d5403a84c7cedf0f23df80 2024-11-22T03:41:02,585 INFO [RS:0;c85114ed5096:42545-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f0af25cc84372b1efb01f18522e2d562#B#compaction#196 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 53.85 MB/second 2024-11-22T03:41:02,585 INFO [RS:0;c85114ed5096:42545-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f0af25cc84372b1efb01f18522e2d562#A#compaction#197 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-22T03:41:02,586 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/.tmp/B/da2c292ff1d049a98249aa78cbe4916b is 50, key is test_row_0/B:col10/1732246860003/Put/seqid=0 2024-11-22T03:41:02,586 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/.tmp/A/4c0b7a3807214367bf4d9d9b94354ab7 is 50, key is test_row_0/A:col10/1732246860003/Put/seqid=0 2024-11-22T03:41:02,603 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/.tmp/B/4f698dbb5cfe4aa69e747bfe37e997d1 is 50, key is test_row_0/B:col10/1732246861752/Put/seqid=0 2024-11-22T03:41:02,619 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742057_1233 (size=12375) 2024-11-22T03:41:02,622 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742056_1232 (size=12375) 2024-11-22T03:41:02,631 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/.tmp/B/da2c292ff1d049a98249aa78cbe4916b as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/B/da2c292ff1d049a98249aa78cbe4916b 2024-11-22T03:41:02,635 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742058_1234 (size=12151) 2024-11-22T03:41:02,637 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=154 (bloomFilter=true), to=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/.tmp/B/4f698dbb5cfe4aa69e747bfe37e997d1 2024-11-22T03:41:02,638 INFO [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in f0af25cc84372b1efb01f18522e2d562/B of f0af25cc84372b1efb01f18522e2d562 into da2c292ff1d049a98249aa78cbe4916b(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T03:41:02,638 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f0af25cc84372b1efb01f18522e2d562: 2024-11-22T03:41:02,638 INFO [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., storeName=f0af25cc84372b1efb01f18522e2d562/B, priority=12, startTime=1732246862552; duration=0sec 2024-11-22T03:41:02,638 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T03:41:02,638 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f0af25cc84372b1efb01f18522e2d562:B 2024-11-22T03:41:02,639 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-22T03:41:02,640 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48242 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-22T03:41:02,640 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HStore(1541): f0af25cc84372b1efb01f18522e2d562/C is initiating minor compaction (all files) 2024-11-22T03:41:02,640 INFO [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f0af25cc84372b1efb01f18522e2d562/C in TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562. 2024-11-22T03:41:02,641 INFO [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/C/444c524511c246a193f089a5386fbc20, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/C/9a5b3dbcec1f45358f56f8b9a360a457, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/C/958da13a301a4354aa1612841c3ce7d4, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/C/04f21fdb166b404380b6ef47531afe11] into tmpdir=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/.tmp, totalSize=47.1 K 2024-11-22T03:41:02,641 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] compactions.Compactor(225): Compacting 444c524511c246a193f089a5386fbc20, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=79, earliestPutTs=1732246857517 2024-11-22T03:41:02,642 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] compactions.Compactor(225): Compacting 9a5b3dbcec1f45358f56f8b9a360a457, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=91, earliestPutTs=1732246858780 2024-11-22T03:41:02,642 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] compactions.Compactor(225): Compacting 958da13a301a4354aa1612841c3ce7d4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1732246858805 2024-11-22T03:41:02,643 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] compactions.Compactor(225): Compacting 04f21fdb166b404380b6ef47531afe11, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1732246860003 2024-11-22T03:41:02,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/.tmp/C/1b3ee846b0af419da2a74cc498c06d9f is 50, key is test_row_0/C:col10/1732246861752/Put/seqid=0 2024-11-22T03:41:02,653 INFO [RS:0;c85114ed5096:42545-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f0af25cc84372b1efb01f18522e2d562#C#compaction#200 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-22T03:41:02,653 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/.tmp/C/0d44541354de4805a659eb9abbcf7a19 is 50, key is test_row_0/C:col10/1732246860003/Put/seqid=0 2024-11-22T03:41:02,654 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742059_1235 (size=12151) 2024-11-22T03:41:02,662 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742060_1236 (size=12375) 2024-11-22T03:41:02,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(8855): Flush requested on f0af25cc84372b1efb01f18522e2d562 2024-11-22T03:41:02,920 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562. as already flushing 2024-11-22T03:41:02,955 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:02,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39226 deadline: 1732246922953, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:02,955 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:02,955 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:02,955 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:41:03,024 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/.tmp/A/4c0b7a3807214367bf4d9d9b94354ab7 as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/A/4c0b7a3807214367bf4d9d9b94354ab7 2024-11-22T03:41:03,029 INFO [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in f0af25cc84372b1efb01f18522e2d562/A of f0af25cc84372b1efb01f18522e2d562 into 4c0b7a3807214367bf4d9d9b94354ab7(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T03:41:03,029 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f0af25cc84372b1efb01f18522e2d562: 2024-11-22T03:41:03,029 INFO [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., storeName=f0af25cc84372b1efb01f18522e2d562/A, priority=12, startTime=1732246862552; duration=0sec 2024-11-22T03:41:03,029 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T03:41:03,029 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f0af25cc84372b1efb01f18522e2d562:A 2024-11-22T03:41:03,054 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=154 (bloomFilter=true), to=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/.tmp/C/1b3ee846b0af419da2a74cc498c06d9f 2024-11-22T03:41:03,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/.tmp/A/7a4710f466d5403a84c7cedf0f23df80 as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/A/7a4710f466d5403a84c7cedf0f23df80 2024-11-22T03:41:03,060 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:03,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39226 deadline: 1732246923058, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:03,060 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:03,060 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:03,060 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:41:03,062 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/A/7a4710f466d5403a84c7cedf0f23df80, entries=150, sequenceid=154, filesize=11.9 K 2024-11-22T03:41:03,063 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/.tmp/B/4f698dbb5cfe4aa69e747bfe37e997d1 as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/B/4f698dbb5cfe4aa69e747bfe37e997d1 2024-11-22T03:41:03,066 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/.tmp/C/0d44541354de4805a659eb9abbcf7a19 as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/C/0d44541354de4805a659eb9abbcf7a19 2024-11-22T03:41:03,067 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/B/4f698dbb5cfe4aa69e747bfe37e997d1, entries=150, sequenceid=154, filesize=11.9 K 2024-11-22T03:41:03,067 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/.tmp/C/1b3ee846b0af419da2a74cc498c06d9f as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/C/1b3ee846b0af419da2a74cc498c06d9f 2024-11-22T03:41:03,070 INFO [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in f0af25cc84372b1efb01f18522e2d562/C of f0af25cc84372b1efb01f18522e2d562 into 0d44541354de4805a659eb9abbcf7a19(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T03:41:03,070 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f0af25cc84372b1efb01f18522e2d562: 2024-11-22T03:41:03,070 INFO [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., storeName=f0af25cc84372b1efb01f18522e2d562/C, priority=12, startTime=1732246862552; duration=0sec 2024-11-22T03:41:03,071 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T03:41:03,071 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f0af25cc84372b1efb01f18522e2d562:C 2024-11-22T03:41:03,072 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/C/1b3ee846b0af419da2a74cc498c06d9f, entries=150, sequenceid=154, filesize=11.9 K 2024-11-22T03:41:03,072 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for f0af25cc84372b1efb01f18522e2d562 in 517ms, sequenceid=154, compaction requested=false 2024-11-22T03:41:03,072 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2603): Flush status journal for f0af25cc84372b1efb01f18522e2d562: 2024-11-22T03:41:03,072 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562. 2024-11-22T03:41:03,072 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=67 2024-11-22T03:41:03,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42627 {}] master.HMaster(4169): Remote procedure done, pid=67 2024-11-22T03:41:03,075 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=67, resume processing ppid=66 2024-11-22T03:41:03,075 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=67, ppid=66, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1300 sec 2024-11-22T03:41:03,077 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=66, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees in 1.1370 sec 2024-11-22T03:41:03,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-22T03:41:03,079 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-22T03:41:03,079 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42627 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-22T03:41:03,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42627 {}] procedure2.ProcedureExecutor(1139): Stored pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees 2024-11-22T03:41:03,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-22T03:41:03,080 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-22T03:41:03,081 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-22T03:41:03,081 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=69, ppid=68, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-22T03:41:03,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-22T03:41:03,233 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42545 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-22T03:41:03,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562. 2024-11-22T03:41:03,233 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2902): Flushing f0af25cc84372b1efb01f18522e2d562 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-22T03:41:03,237 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/.tmp/A/576e023f019d4aaeaddfcdcc40ef8a24 is 50, key is test_row_0/A:col10/1732246862935/Put/seqid=0 2024-11-22T03:41:03,242 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742061_1237 (size=12151) 2024-11-22T03:41:03,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(8855): Flush requested on f0af25cc84372b1efb01f18522e2d562 2024-11-22T03:41:03,270 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562. as already flushing 2024-11-22T03:41:03,348 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:03,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39226 deadline: 1732246923345, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:03,348 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:03,348 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:03,348 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:41:03,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-22T03:41:03,460 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:03,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39226 deadline: 1732246923459, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:03,460 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:03,460 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:03,460 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:41:03,641 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=170 (bloomFilter=true), to=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/.tmp/A/576e023f019d4aaeaddfcdcc40ef8a24 2024-11-22T03:41:03,648 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/.tmp/B/691c750a61904a7fa9e41cb5324a1dba is 50, key is test_row_0/B:col10/1732246862935/Put/seqid=0 2024-11-22T03:41:03,652 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742062_1238 (size=12151) 2024-11-22T03:41:03,669 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:03,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39226 deadline: 1732246923668, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:03,670 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:03,670 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:03,670 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:41:03,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-22T03:41:03,980 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:03,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39226 deadline: 1732246923979, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:03,981 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:03,981 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:03,981 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:41:04,053 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=170 (bloomFilter=true), to=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/.tmp/B/691c750a61904a7fa9e41cb5324a1dba 2024-11-22T03:41:04,061 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/.tmp/C/7b619c6f759a4e05b8234045512bb775 is 50, key is test_row_0/C:col10/1732246862935/Put/seqid=0 2024-11-22T03:41:04,065 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742063_1239 (size=12151) 2024-11-22T03:41:04,181 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:04,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39216 deadline: 1732246924178, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:04,182 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:04,182 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:04,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39228 deadline: 1732246924178, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:04,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39154 deadline: 1732246924178, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:04,182 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to c85114ed5096,42545,1732246786482 for 'test_row_1' in f0af25cc84372b1efb01f18522e2d562 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4178 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-22T03:41:04,183 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:04,183 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:04,182 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to c85114ed5096,42545,1732246786482 for 'test_row_0' in f0af25cc84372b1efb01f18522e2d562 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4178 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-22T03:41:04,183 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:41:04,182 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to c85114ed5096,42545,1732246786482 for 'test_row_1' in f0af25cc84372b1efb01f18522e2d562 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4173 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-22T03:41:04,183 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:04,183 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:04,183 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:04,183 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:41:04,183 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:04,183 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:41:04,191 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:04,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39242 deadline: 1732246924189, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:04,191 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to c85114ed5096,42545,1732246786482 for 'test_row_1' in f0af25cc84372b1efb01f18522e2d562 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4186 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-22T03:41:04,192 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:04,192 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:04,192 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:41:04,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-22T03:41:04,465 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=170 (bloomFilter=true), to=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/.tmp/C/7b619c6f759a4e05b8234045512bb775 2024-11-22T03:41:04,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/.tmp/A/576e023f019d4aaeaddfcdcc40ef8a24 as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/A/576e023f019d4aaeaddfcdcc40ef8a24 2024-11-22T03:41:04,473 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/A/576e023f019d4aaeaddfcdcc40ef8a24, entries=150, sequenceid=170, filesize=11.9 K 2024-11-22T03:41:04,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/.tmp/B/691c750a61904a7fa9e41cb5324a1dba as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/B/691c750a61904a7fa9e41cb5324a1dba 2024-11-22T03:41:04,477 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/B/691c750a61904a7fa9e41cb5324a1dba, entries=150, sequenceid=170, filesize=11.9 K 2024-11-22T03:41:04,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/.tmp/C/7b619c6f759a4e05b8234045512bb775 as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/C/7b619c6f759a4e05b8234045512bb775 2024-11-22T03:41:04,481 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/C/7b619c6f759a4e05b8234045512bb775, entries=150, sequenceid=170, filesize=11.9 K 2024-11-22T03:41:04,482 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for f0af25cc84372b1efb01f18522e2d562 in 1249ms, sequenceid=170, compaction requested=true 2024-11-22T03:41:04,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2603): Flush status journal for f0af25cc84372b1efb01f18522e2d562: 2024-11-22T03:41:04,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562. 2024-11-22T03:41:04,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=69 2024-11-22T03:41:04,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42627 {}] master.HMaster(4169): Remote procedure done, pid=69 2024-11-22T03:41:04,485 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=69, resume processing ppid=68 2024-11-22T03:41:04,485 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=69, ppid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4020 sec 2024-11-22T03:41:04,486 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees in 1.4060 sec 2024-11-22T03:41:04,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(8855): Flush requested on f0af25cc84372b1efb01f18522e2d562 2024-11-22T03:41:04,491 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-22T03:41:04,491 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f0af25cc84372b1efb01f18522e2d562 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-22T03:41:04,495 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/.tmp/A/6395378272cc4904a0e8928dfec2c720 is 50, key is test_row_0/A:col10/1732246863321/Put/seqid=0 2024-11-22T03:41:04,510 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742064_1240 (size=14541) 2024-11-22T03:41:04,530 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:04,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39226 deadline: 1732246924527, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:04,530 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:04,530 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:04,530 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:41:04,647 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:04,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39226 deadline: 1732246924638, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:04,647 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:04,647 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:04,647 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:41:04,863 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:04,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39226 deadline: 1732246924859, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:04,863 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:04,863 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:04,863 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:41:04,911 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=194 (bloomFilter=true), to=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/.tmp/A/6395378272cc4904a0e8928dfec2c720 2024-11-22T03:41:04,917 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/.tmp/B/96f0fc7b078a4512bc5f4705da404a2e is 50, key is test_row_0/B:col10/1732246863321/Put/seqid=0 2024-11-22T03:41:04,921 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742065_1241 (size=12151) 2024-11-22T03:41:05,171 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:05,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39226 deadline: 1732246925169, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:05,171 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:05,171 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:05,171 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:41:05,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-22T03:41:05,239 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-22T03:41:05,239 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42627 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-22T03:41:05,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42627 {}] procedure2.ProcedureExecutor(1139): Stored pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees 2024-11-22T03:41:05,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-22T03:41:05,240 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-22T03:41:05,241 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-22T03:41:05,241 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=71, ppid=70, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-22T03:41:05,322 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=194 (bloomFilter=true), to=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/.tmp/B/96f0fc7b078a4512bc5f4705da404a2e 2024-11-22T03:41:05,329 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/.tmp/C/aaeef659ae27433f80d35920dadb86ce is 50, key is test_row_0/C:col10/1732246863321/Put/seqid=0 2024-11-22T03:41:05,332 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742066_1242 (size=12151) 2024-11-22T03:41:05,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-22T03:41:05,393 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42545 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-22T03:41:05,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562. 2024-11-22T03:41:05,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562. as already flushing 2024-11-22T03:41:05,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562. 2024-11-22T03:41:05,393 ERROR [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => f0af25cc84372b1efb01f18522e2d562, NAME => 'TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:41:05,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => f0af25cc84372b1efb01f18522e2d562, NAME => 'TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:41:05,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42627 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f0af25cc84372b1efb01f18522e2d562, NAME => 'TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f0af25cc84372b1efb01f18522e2d562, NAME => 'TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:41:05,546 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42545 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-22T03:41:05,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562. 2024-11-22T03:41:05,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562. as already flushing 2024-11-22T03:41:05,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562. 2024-11-22T03:41:05,546 ERROR [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => f0af25cc84372b1efb01f18522e2d562, NAME => 'TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:41:05,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => f0af25cc84372b1efb01f18522e2d562, NAME => 'TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:41:05,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42627 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f0af25cc84372b1efb01f18522e2d562, NAME => 'TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f0af25cc84372b1efb01f18522e2d562, NAME => 'TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:41:05,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-22T03:41:05,681 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:05,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39226 deadline: 1732246925679, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:05,681 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:05,681 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:05,681 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:41:05,699 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42545 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-22T03:41:05,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562. 2024-11-22T03:41:05,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562. as already flushing 2024-11-22T03:41:05,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562. 2024-11-22T03:41:05,699 ERROR [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => f0af25cc84372b1efb01f18522e2d562, NAME => 'TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:41:05,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => f0af25cc84372b1efb01f18522e2d562, NAME => 'TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:41:05,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42627 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f0af25cc84372b1efb01f18522e2d562, NAME => 'TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f0af25cc84372b1efb01f18522e2d562, NAME => 'TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:41:05,733 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=194 (bloomFilter=true), to=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/.tmp/C/aaeef659ae27433f80d35920dadb86ce 2024-11-22T03:41:05,737 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/.tmp/A/6395378272cc4904a0e8928dfec2c720 as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/A/6395378272cc4904a0e8928dfec2c720 2024-11-22T03:41:05,741 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/A/6395378272cc4904a0e8928dfec2c720, entries=200, sequenceid=194, filesize=14.2 K 2024-11-22T03:41:05,742 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/.tmp/B/96f0fc7b078a4512bc5f4705da404a2e as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/B/96f0fc7b078a4512bc5f4705da404a2e 2024-11-22T03:41:05,745 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/B/96f0fc7b078a4512bc5f4705da404a2e, entries=150, sequenceid=194, filesize=11.9 K 2024-11-22T03:41:05,746 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/.tmp/C/aaeef659ae27433f80d35920dadb86ce as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/C/aaeef659ae27433f80d35920dadb86ce 2024-11-22T03:41:05,750 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/C/aaeef659ae27433f80d35920dadb86ce, entries=150, sequenceid=194, filesize=11.9 K 2024-11-22T03:41:05,751 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for f0af25cc84372b1efb01f18522e2d562 in 1260ms, sequenceid=194, compaction requested=true 2024-11-22T03:41:05,751 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f0af25cc84372b1efb01f18522e2d562: 2024-11-22T03:41:05,751 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f0af25cc84372b1efb01f18522e2d562:A, priority=-2147483648, current under compaction store size is 1 2024-11-22T03:41:05,751 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T03:41:05,751 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-22T03:41:05,751 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f0af25cc84372b1efb01f18522e2d562:B, priority=-2147483648, current under compaction store size is 2 2024-11-22T03:41:05,751 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T03:41:05,751 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-22T03:41:05,751 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f0af25cc84372b1efb01f18522e2d562:C, priority=-2147483648, current under compaction store size is 3 2024-11-22T03:41:05,751 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T03:41:05,753 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 51218 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-22T03:41:05,753 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48828 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-22T03:41:05,753 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HStore(1541): f0af25cc84372b1efb01f18522e2d562/B is initiating minor compaction (all files) 2024-11-22T03:41:05,753 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.HStore(1541): f0af25cc84372b1efb01f18522e2d562/A is initiating minor compaction (all files) 2024-11-22T03:41:05,753 INFO [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f0af25cc84372b1efb01f18522e2d562/A in TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562. 2024-11-22T03:41:05,753 INFO [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f0af25cc84372b1efb01f18522e2d562/B in TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562. 2024-11-22T03:41:05,753 INFO [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/A/4c0b7a3807214367bf4d9d9b94354ab7, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/A/7a4710f466d5403a84c7cedf0f23df80, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/A/576e023f019d4aaeaddfcdcc40ef8a24, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/A/6395378272cc4904a0e8928dfec2c720] into tmpdir=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/.tmp, totalSize=50.0 K 2024-11-22T03:41:05,753 INFO [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/B/da2c292ff1d049a98249aa78cbe4916b, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/B/4f698dbb5cfe4aa69e747bfe37e997d1, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/B/691c750a61904a7fa9e41cb5324a1dba, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/B/96f0fc7b078a4512bc5f4705da404a2e] into tmpdir=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/.tmp, totalSize=47.7 K 2024-11-22T03:41:05,754 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] compactions.Compactor(225): Compacting da2c292ff1d049a98249aa78cbe4916b, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1732246860003 2024-11-22T03:41:05,754 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4c0b7a3807214367bf4d9d9b94354ab7, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1732246860003 2024-11-22T03:41:05,754 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7a4710f466d5403a84c7cedf0f23df80, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1732246861752 2024-11-22T03:41:05,754 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] compactions.Compactor(225): Compacting 4f698dbb5cfe4aa69e747bfe37e997d1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1732246861752 2024-11-22T03:41:05,754 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] compactions.Compactor(225): Compacting 576e023f019d4aaeaddfcdcc40ef8a24, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1732246862935 2024-11-22T03:41:05,754 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] compactions.Compactor(225): Compacting 691c750a61904a7fa9e41cb5324a1dba, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1732246862935 2024-11-22T03:41:05,755 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] compactions.Compactor(225): Compacting 96f0fc7b078a4512bc5f4705da404a2e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1732246863321 2024-11-22T03:41:05,755 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6395378272cc4904a0e8928dfec2c720, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1732246863321 2024-11-22T03:41:05,763 INFO [RS:0;c85114ed5096:42545-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f0af25cc84372b1efb01f18522e2d562#B#compaction#208 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 53.85 MB/second 2024-11-22T03:41:05,764 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/.tmp/B/13789e4827384b3bb7b113f1ddaea36e is 50, key is test_row_0/B:col10/1732246863321/Put/seqid=0 2024-11-22T03:41:05,764 INFO [RS:0;c85114ed5096:42545-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f0af25cc84372b1efb01f18522e2d562#A#compaction#207 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-22T03:41:05,765 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/.tmp/A/c0e83b1eb5334578a7e6871eae8a2565 is 50, key is test_row_0/A:col10/1732246863321/Put/seqid=0 2024-11-22T03:41:05,787 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742067_1243 (size=12561) 2024-11-22T03:41:05,788 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742068_1244 (size=12561) 2024-11-22T03:41:05,852 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42545 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-22T03:41:05,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562. 2024-11-22T03:41:05,852 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2902): Flushing f0af25cc84372b1efb01f18522e2d562 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-22T03:41:05,857 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/.tmp/A/e7dfeff5273b4f4f83624040bed8557d is 50, key is test_row_0/A:col10/1732246864513/Put/seqid=0 2024-11-22T03:41:05,866 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742069_1245 (size=12151) 2024-11-22T03:41:05,867 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=206 (bloomFilter=true), to=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/.tmp/A/e7dfeff5273b4f4f83624040bed8557d 2024-11-22T03:41:05,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-22T03:41:05,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/.tmp/B/0ef60703a1e140c1a14e6a9717b4b42a is 50, key is test_row_0/B:col10/1732246864513/Put/seqid=0 2024-11-22T03:41:05,892 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742070_1246 (size=12151) 2024-11-22T03:41:05,895 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=206 (bloomFilter=true), to=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/.tmp/B/0ef60703a1e140c1a14e6a9717b4b42a 2024-11-22T03:41:05,911 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/.tmp/C/aa5b0362060e4bb78740f2b77f4b413c is 50, key is test_row_0/C:col10/1732246864513/Put/seqid=0 2024-11-22T03:41:05,927 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742071_1247 (size=12151) 2024-11-22T03:41:06,193 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/.tmp/B/13789e4827384b3bb7b113f1ddaea36e as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/B/13789e4827384b3bb7b113f1ddaea36e 2024-11-22T03:41:06,193 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/.tmp/A/c0e83b1eb5334578a7e6871eae8a2565 as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/A/c0e83b1eb5334578a7e6871eae8a2565 2024-11-22T03:41:06,198 INFO [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in f0af25cc84372b1efb01f18522e2d562/A of f0af25cc84372b1efb01f18522e2d562 into c0e83b1eb5334578a7e6871eae8a2565(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T03:41:06,198 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f0af25cc84372b1efb01f18522e2d562: 2024-11-22T03:41:06,198 INFO [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., storeName=f0af25cc84372b1efb01f18522e2d562/A, priority=12, startTime=1732246865751; duration=0sec 2024-11-22T03:41:06,198 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T03:41:06,198 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f0af25cc84372b1efb01f18522e2d562:A 2024-11-22T03:41:06,198 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-22T03:41:06,199 INFO [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in f0af25cc84372b1efb01f18522e2d562/B of f0af25cc84372b1efb01f18522e2d562 into 13789e4827384b3bb7b113f1ddaea36e(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T03:41:06,199 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f0af25cc84372b1efb01f18522e2d562: 2024-11-22T03:41:06,199 INFO [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., storeName=f0af25cc84372b1efb01f18522e2d562/B, priority=12, startTime=1732246865751; duration=0sec 2024-11-22T03:41:06,199 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T03:41:06,199 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f0af25cc84372b1efb01f18522e2d562:B 2024-11-22T03:41:06,200 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48828 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-22T03:41:06,200 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.HStore(1541): f0af25cc84372b1efb01f18522e2d562/C is initiating minor compaction (all files) 2024-11-22T03:41:06,200 INFO [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f0af25cc84372b1efb01f18522e2d562/C in TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562. 2024-11-22T03:41:06,200 INFO [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/C/0d44541354de4805a659eb9abbcf7a19, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/C/1b3ee846b0af419da2a74cc498c06d9f, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/C/7b619c6f759a4e05b8234045512bb775, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/C/aaeef659ae27433f80d35920dadb86ce] into tmpdir=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/.tmp, totalSize=47.7 K 2024-11-22T03:41:06,201 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0d44541354de4805a659eb9abbcf7a19, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1732246860003 2024-11-22T03:41:06,201 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1b3ee846b0af419da2a74cc498c06d9f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1732246861752 2024-11-22T03:41:06,201 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7b619c6f759a4e05b8234045512bb775, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1732246862935 2024-11-22T03:41:06,202 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] compactions.Compactor(225): Compacting aaeef659ae27433f80d35920dadb86ce, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1732246863321 2024-11-22T03:41:06,216 INFO [RS:0;c85114ed5096:42545-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f0af25cc84372b1efb01f18522e2d562#C#compaction#212 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-22T03:41:06,217 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/.tmp/C/d02a0d89b3d04b78b59b7da1676d7770 is 50, key is test_row_0/C:col10/1732246863321/Put/seqid=0 2024-11-22T03:41:06,226 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742072_1248 (size=12561) 2024-11-22T03:41:06,238 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/.tmp/C/d02a0d89b3d04b78b59b7da1676d7770 as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/C/d02a0d89b3d04b78b59b7da1676d7770 2024-11-22T03:41:06,243 INFO [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in f0af25cc84372b1efb01f18522e2d562/C of f0af25cc84372b1efb01f18522e2d562 into d02a0d89b3d04b78b59b7da1676d7770(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T03:41:06,244 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f0af25cc84372b1efb01f18522e2d562: 2024-11-22T03:41:06,244 INFO [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., storeName=f0af25cc84372b1efb01f18522e2d562/C, priority=12, startTime=1732246865751; duration=0sec 2024-11-22T03:41:06,244 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T03:41:06,244 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f0af25cc84372b1efb01f18522e2d562:C 2024-11-22T03:41:06,327 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=206 (bloomFilter=true), to=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/.tmp/C/aa5b0362060e4bb78740f2b77f4b413c 2024-11-22T03:41:06,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/.tmp/A/e7dfeff5273b4f4f83624040bed8557d as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/A/e7dfeff5273b4f4f83624040bed8557d 2024-11-22T03:41:06,335 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/A/e7dfeff5273b4f4f83624040bed8557d, entries=150, sequenceid=206, filesize=11.9 K 2024-11-22T03:41:06,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/.tmp/B/0ef60703a1e140c1a14e6a9717b4b42a as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/B/0ef60703a1e140c1a14e6a9717b4b42a 2024-11-22T03:41:06,339 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/B/0ef60703a1e140c1a14e6a9717b4b42a, entries=150, sequenceid=206, filesize=11.9 K 2024-11-22T03:41:06,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/.tmp/C/aa5b0362060e4bb78740f2b77f4b413c as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/C/aa5b0362060e4bb78740f2b77f4b413c 2024-11-22T03:41:06,350 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/C/aa5b0362060e4bb78740f2b77f4b413c, entries=150, sequenceid=206, filesize=11.9 K 2024-11-22T03:41:06,350 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=0 B/0 for f0af25cc84372b1efb01f18522e2d562 in 498ms, sequenceid=206, compaction requested=false 2024-11-22T03:41:06,350 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2603): Flush status journal for f0af25cc84372b1efb01f18522e2d562: 2024-11-22T03:41:06,350 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562. 2024-11-22T03:41:06,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=71 2024-11-22T03:41:06,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42627 {}] master.HMaster(4169): Remote procedure done, pid=71 2024-11-22T03:41:06,353 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=71, resume processing ppid=70 2024-11-22T03:41:06,353 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=71, ppid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1110 sec 2024-11-22T03:41:06,355 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees in 1.1150 sec 2024-11-22T03:41:06,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-22T03:41:06,379 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-22T03:41:06,379 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42627 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-22T03:41:06,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42627 {}] procedure2.ProcedureExecutor(1139): Stored pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees 2024-11-22T03:41:06,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-22T03:41:06,381 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-22T03:41:06,382 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-22T03:41:06,382 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=73, ppid=72, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-22T03:41:06,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-22T03:41:06,538 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42545 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-22T03:41:06,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562. 2024-11-22T03:41:06,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2603): Flush status journal for f0af25cc84372b1efb01f18522e2d562: 2024-11-22T03:41:06,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562. 2024-11-22T03:41:06,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=73 2024-11-22T03:41:06,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42627 {}] master.HMaster(4169): Remote procedure done, pid=73 2024-11-22T03:41:06,541 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=73, resume processing ppid=72 2024-11-22T03:41:06,541 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=73, ppid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 158 msec 2024-11-22T03:41:06,543 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees in 163 msec 2024-11-22T03:41:06,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-22T03:41:06,699 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-22T03:41:06,699 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42627 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-22T03:41:06,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42627 {}] procedure2.ProcedureExecutor(1139): Stored pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees 2024-11-22T03:41:06,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-22T03:41:06,700 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-22T03:41:06,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(8855): Flush requested on f0af25cc84372b1efb01f18522e2d562 2024-11-22T03:41:06,701 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-22T03:41:06,701 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f0af25cc84372b1efb01f18522e2d562 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-22T03:41:06,701 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-22T03:41:06,701 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=75, ppid=74, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-22T03:41:06,707 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/.tmp/A/21c14fc0778a4e34988a98ff861410ad is 50, key is test_row_0/A:col10/1732246866693/Put/seqid=0 2024-11-22T03:41:06,722 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742073_1249 (size=14541) 2024-11-22T03:41:06,786 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:06,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39226 deadline: 1732246926784, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:06,787 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:06,787 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:06,787 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:41:06,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-22T03:41:06,852 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42545 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-22T03:41:06,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562. 2024-11-22T03:41:06,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562. as already flushing 2024-11-22T03:41:06,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562. 2024-11-22T03:41:06,853 ERROR [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => f0af25cc84372b1efb01f18522e2d562, NAME => 'TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:41:06,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => f0af25cc84372b1efb01f18522e2d562, NAME => 'TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:41:06,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42627 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f0af25cc84372b1efb01f18522e2d562, NAME => 'TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f0af25cc84372b1efb01f18522e2d562, NAME => 'TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:41:06,890 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:06,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39226 deadline: 1732246926889, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:06,891 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:06,891 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:06,891 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:41:07,005 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42545 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-22T03:41:07,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562. 2024-11-22T03:41:07,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562. as already flushing 2024-11-22T03:41:07,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562. 2024-11-22T03:41:07,005 ERROR [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => f0af25cc84372b1efb01f18522e2d562, NAME => 'TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:41:07,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => f0af25cc84372b1efb01f18522e2d562, NAME => 'TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:41:07,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42627 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f0af25cc84372b1efb01f18522e2d562, NAME => 'TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f0af25cc84372b1efb01f18522e2d562, NAME => 'TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:41:07,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-22T03:41:07,101 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:07,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39226 deadline: 1732246927099, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:07,101 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:07,101 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:07,101 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:41:07,122 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=221 (bloomFilter=true), to=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/.tmp/A/21c14fc0778a4e34988a98ff861410ad 2024-11-22T03:41:07,128 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/.tmp/B/a529d6e5713d48b1855ee23456d3a775 is 50, key is test_row_0/B:col10/1732246866693/Put/seqid=0 2024-11-22T03:41:07,132 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742074_1250 (size=12151) 2024-11-22T03:41:07,158 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42545 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-22T03:41:07,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562. 2024-11-22T03:41:07,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562. as already flushing 2024-11-22T03:41:07,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562. 2024-11-22T03:41:07,158 ERROR [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => f0af25cc84372b1efb01f18522e2d562, NAME => 'TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:41:07,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => f0af25cc84372b1efb01f18522e2d562, NAME => 'TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:41:07,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42627 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f0af25cc84372b1efb01f18522e2d562, NAME => 'TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f0af25cc84372b1efb01f18522e2d562, NAME => 'TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:41:07,311 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42545 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-22T03:41:07,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562. 2024-11-22T03:41:07,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562. as already flushing 2024-11-22T03:41:07,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562. 2024-11-22T03:41:07,311 ERROR [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => f0af25cc84372b1efb01f18522e2d562, NAME => 'TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:41:07,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => f0af25cc84372b1efb01f18522e2d562, NAME => 'TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:41:07,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42627 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f0af25cc84372b1efb01f18522e2d562, NAME => 'TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f0af25cc84372b1efb01f18522e2d562, NAME => 'TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:41:07,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-22T03:41:07,411 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:07,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39226 deadline: 1732246927409, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:07,411 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:07,411 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:07,411 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:41:07,463 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42545 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-22T03:41:07,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562. 2024-11-22T03:41:07,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562. as already flushing 2024-11-22T03:41:07,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562. 2024-11-22T03:41:07,464 ERROR [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => f0af25cc84372b1efb01f18522e2d562, NAME => 'TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:41:07,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => f0af25cc84372b1efb01f18522e2d562, NAME => 'TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:41:07,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42627 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f0af25cc84372b1efb01f18522e2d562, NAME => 'TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f0af25cc84372b1efb01f18522e2d562, NAME => 'TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:41:07,533 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=221 (bloomFilter=true), to=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/.tmp/B/a529d6e5713d48b1855ee23456d3a775 2024-11-22T03:41:07,539 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/.tmp/C/e94c61045368464bb576a5e022cf9ae0 is 50, key is test_row_0/C:col10/1732246866693/Put/seqid=0 2024-11-22T03:41:07,543 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742075_1251 (size=12151) 2024-11-22T03:41:07,616 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42545 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-22T03:41:07,616 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562. 2024-11-22T03:41:07,616 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562. as already flushing 2024-11-22T03:41:07,616 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562. 2024-11-22T03:41:07,616 ERROR [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => f0af25cc84372b1efb01f18522e2d562, NAME => 'TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:41:07,616 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => f0af25cc84372b1efb01f18522e2d562, NAME => 'TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:41:07,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42627 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f0af25cc84372b1efb01f18522e2d562, NAME => 'TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f0af25cc84372b1efb01f18522e2d562, NAME => 'TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:41:07,769 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42545 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-22T03:41:07,769 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562. 2024-11-22T03:41:07,770 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562. as already flushing 2024-11-22T03:41:07,770 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562. 2024-11-22T03:41:07,770 ERROR [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => f0af25cc84372b1efb01f18522e2d562, NAME => 'TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:41:07,770 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => f0af25cc84372b1efb01f18522e2d562, NAME => 'TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:41:07,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42627 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f0af25cc84372b1efb01f18522e2d562, NAME => 'TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f0af25cc84372b1efb01f18522e2d562, NAME => 'TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:41:07,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-22T03:41:07,921 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:07,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39226 deadline: 1732246927919, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:07,921 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:07,921 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:07,921 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:41:07,922 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42545 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-22T03:41:07,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562. 2024-11-22T03:41:07,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562. as already flushing 2024-11-22T03:41:07,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562. 2024-11-22T03:41:07,922 ERROR [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => f0af25cc84372b1efb01f18522e2d562, NAME => 'TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:41:07,923 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => f0af25cc84372b1efb01f18522e2d562, NAME => 'TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:41:07,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42627 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f0af25cc84372b1efb01f18522e2d562, NAME => 'TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f0af25cc84372b1efb01f18522e2d562, NAME => 'TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:41:07,944 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=221 (bloomFilter=true), to=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/.tmp/C/e94c61045368464bb576a5e022cf9ae0 2024-11-22T03:41:07,947 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/.tmp/A/21c14fc0778a4e34988a98ff861410ad as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/A/21c14fc0778a4e34988a98ff861410ad 2024-11-22T03:41:07,951 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/A/21c14fc0778a4e34988a98ff861410ad, entries=200, sequenceid=221, filesize=14.2 K 2024-11-22T03:41:07,951 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/.tmp/B/a529d6e5713d48b1855ee23456d3a775 as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/B/a529d6e5713d48b1855ee23456d3a775 2024-11-22T03:41:07,955 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/B/a529d6e5713d48b1855ee23456d3a775, entries=150, sequenceid=221, filesize=11.9 K 2024-11-22T03:41:07,956 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/.tmp/C/e94c61045368464bb576a5e022cf9ae0 as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/C/e94c61045368464bb576a5e022cf9ae0 2024-11-22T03:41:07,959 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/C/e94c61045368464bb576a5e022cf9ae0, entries=150, sequenceid=221, filesize=11.9 K 2024-11-22T03:41:07,960 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for f0af25cc84372b1efb01f18522e2d562 in 1259ms, sequenceid=221, compaction requested=true 2024-11-22T03:41:07,960 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f0af25cc84372b1efb01f18522e2d562: 2024-11-22T03:41:07,960 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f0af25cc84372b1efb01f18522e2d562:A, priority=-2147483648, current under compaction store size is 1 2024-11-22T03:41:07,960 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T03:41:07,960 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T03:41:07,960 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f0af25cc84372b1efb01f18522e2d562:B, priority=-2147483648, current under compaction store size is 2 2024-11-22T03:41:07,960 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T03:41:07,961 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T03:41:07,961 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f0af25cc84372b1efb01f18522e2d562:C, priority=-2147483648, current under compaction store size is 3 2024-11-22T03:41:07,961 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T03:41:07,962 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T03:41:07,962 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39253 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T03:41:07,962 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.HStore(1541): f0af25cc84372b1efb01f18522e2d562/A is initiating minor compaction (all files) 2024-11-22T03:41:07,962 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HStore(1541): f0af25cc84372b1efb01f18522e2d562/B is initiating minor compaction (all files) 2024-11-22T03:41:07,962 INFO [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f0af25cc84372b1efb01f18522e2d562/A in TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562. 2024-11-22T03:41:07,962 INFO [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/A/c0e83b1eb5334578a7e6871eae8a2565, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/A/e7dfeff5273b4f4f83624040bed8557d, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/A/21c14fc0778a4e34988a98ff861410ad] into tmpdir=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/.tmp, totalSize=38.3 K 2024-11-22T03:41:07,962 INFO [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f0af25cc84372b1efb01f18522e2d562/B in TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562. 2024-11-22T03:41:07,962 INFO [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/B/13789e4827384b3bb7b113f1ddaea36e, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/B/0ef60703a1e140c1a14e6a9717b4b42a, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/B/a529d6e5713d48b1855ee23456d3a775] into tmpdir=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/.tmp, totalSize=36.0 K 2024-11-22T03:41:07,962 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] compactions.Compactor(225): Compacting c0e83b1eb5334578a7e6871eae8a2565, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1732246863321 2024-11-22T03:41:07,962 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] compactions.Compactor(225): Compacting 13789e4827384b3bb7b113f1ddaea36e, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1732246863321 2024-11-22T03:41:07,963 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] compactions.Compactor(225): Compacting 0ef60703a1e140c1a14e6a9717b4b42a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=206, earliestPutTs=1732246864513 2024-11-22T03:41:07,963 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] compactions.Compactor(225): Compacting e7dfeff5273b4f4f83624040bed8557d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=206, earliestPutTs=1732246864513 2024-11-22T03:41:07,963 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] compactions.Compactor(225): Compacting a529d6e5713d48b1855ee23456d3a775, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=221, earliestPutTs=1732246866693 2024-11-22T03:41:07,963 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] compactions.Compactor(225): Compacting 21c14fc0778a4e34988a98ff861410ad, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=221, earliestPutTs=1732246866693 2024-11-22T03:41:07,979 INFO [RS:0;c85114ed5096:42545-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f0af25cc84372b1efb01f18522e2d562#A#compaction#216 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-22T03:41:07,979 INFO [RS:0;c85114ed5096:42545-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f0af25cc84372b1efb01f18522e2d562#B#compaction#217 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-22T03:41:07,980 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/.tmp/A/d2bdfe5f6b534c8c9482bf20d51afc16 is 50, key is test_row_0/A:col10/1732246866693/Put/seqid=0 2024-11-22T03:41:07,980 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/.tmp/B/9d3c752c288140f99b9a2e93e576d3cd is 50, key is test_row_0/B:col10/1732246866693/Put/seqid=0 2024-11-22T03:41:07,988 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742076_1252 (size=12663) 2024-11-22T03:41:07,993 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/.tmp/A/d2bdfe5f6b534c8c9482bf20d51afc16 as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/A/d2bdfe5f6b534c8c9482bf20d51afc16 2024-11-22T03:41:07,994 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742077_1253 (size=12663) 2024-11-22T03:41:07,999 INFO [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f0af25cc84372b1efb01f18522e2d562/A of f0af25cc84372b1efb01f18522e2d562 into d2bdfe5f6b534c8c9482bf20d51afc16(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T03:41:07,999 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f0af25cc84372b1efb01f18522e2d562: 2024-11-22T03:41:07,999 INFO [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., storeName=f0af25cc84372b1efb01f18522e2d562/A, priority=13, startTime=1732246867960; duration=0sec 2024-11-22T03:41:07,999 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T03:41:07,999 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f0af25cc84372b1efb01f18522e2d562:A 2024-11-22T03:41:07,999 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T03:41:08,001 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T03:41:08,001 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.HStore(1541): f0af25cc84372b1efb01f18522e2d562/C is initiating minor compaction (all files) 2024-11-22T03:41:08,001 INFO [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f0af25cc84372b1efb01f18522e2d562/C in TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562. 2024-11-22T03:41:08,001 INFO [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/C/d02a0d89b3d04b78b59b7da1676d7770, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/C/aa5b0362060e4bb78740f2b77f4b413c, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/C/e94c61045368464bb576a5e022cf9ae0] into tmpdir=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/.tmp, totalSize=36.0 K 2024-11-22T03:41:08,002 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] compactions.Compactor(225): Compacting d02a0d89b3d04b78b59b7da1676d7770, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1732246863321 2024-11-22T03:41:08,002 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] compactions.Compactor(225): Compacting aa5b0362060e4bb78740f2b77f4b413c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=206, earliestPutTs=1732246864513 2024-11-22T03:41:08,003 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] compactions.Compactor(225): Compacting e94c61045368464bb576a5e022cf9ae0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=221, earliestPutTs=1732246866693 2024-11-22T03:41:08,013 INFO [RS:0;c85114ed5096:42545-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f0af25cc84372b1efb01f18522e2d562#C#compaction#218 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-22T03:41:08,014 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/.tmp/C/8cc3162719944edea3667433d5827b83 is 50, key is test_row_0/C:col10/1732246866693/Put/seqid=0 2024-11-22T03:41:08,019 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742078_1254 (size=12663) 2024-11-22T03:41:08,024 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/.tmp/C/8cc3162719944edea3667433d5827b83 as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/C/8cc3162719944edea3667433d5827b83 2024-11-22T03:41:08,030 INFO [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f0af25cc84372b1efb01f18522e2d562/C of f0af25cc84372b1efb01f18522e2d562 into 8cc3162719944edea3667433d5827b83(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T03:41:08,030 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f0af25cc84372b1efb01f18522e2d562: 2024-11-22T03:41:08,030 INFO [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., storeName=f0af25cc84372b1efb01f18522e2d562/C, priority=13, startTime=1732246867961; duration=0sec 2024-11-22T03:41:08,030 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T03:41:08,030 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f0af25cc84372b1efb01f18522e2d562:C 2024-11-22T03:41:08,075 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42545 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-22T03:41:08,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562. 2024-11-22T03:41:08,075 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2902): Flushing f0af25cc84372b1efb01f18522e2d562 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-22T03:41:08,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/.tmp/A/d040c025d62e44988c9596e31e8b49e2 is 50, key is test_row_0/A:col10/1732246866728/Put/seqid=0 2024-11-22T03:41:08,086 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742079_1255 (size=12151) 2024-11-22T03:41:08,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(8855): Flush requested on f0af25cc84372b1efb01f18522e2d562 2024-11-22T03:41:08,201 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562. as already flushing 2024-11-22T03:41:08,214 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:08,214 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:08,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39228 deadline: 1732246928209, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:08,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39216 deadline: 1732246928209, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:08,215 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to c85114ed5096,42545,1732246786482 for 'test_row_0' in f0af25cc84372b1efb01f18522e2d562 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8211 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-22T03:41:08,215 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:08,215 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:08,215 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:41:08,215 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to c85114ed5096,42545,1732246786482 for 'test_row_1' in f0af25cc84372b1efb01f18522e2d562 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8211 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-22T03:41:08,216 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:08,216 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:08,216 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:41:08,219 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:08,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39242 deadline: 1732246928215, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:08,219 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:08,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39154 deadline: 1732246928215, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:08,219 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:08,219 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:08,219 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:08,219 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:41:08,219 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:08,219 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:41:08,329 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:08,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39242 deadline: 1732246928328, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:08,330 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:08,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39154 deadline: 1732246928328, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:08,330 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:08,330 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:08,330 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:41:08,330 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:08,330 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:08,330 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:41:08,400 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/.tmp/B/9d3c752c288140f99b9a2e93e576d3cd as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/B/9d3c752c288140f99b9a2e93e576d3cd 2024-11-22T03:41:08,404 INFO [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f0af25cc84372b1efb01f18522e2d562/B of f0af25cc84372b1efb01f18522e2d562 into 9d3c752c288140f99b9a2e93e576d3cd(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T03:41:08,404 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f0af25cc84372b1efb01f18522e2d562: 2024-11-22T03:41:08,404 INFO [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., storeName=f0af25cc84372b1efb01f18522e2d562/B, priority=13, startTime=1732246867960; duration=0sec 2024-11-22T03:41:08,404 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T03:41:08,404 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f0af25cc84372b1efb01f18522e2d562:B 2024-11-22T03:41:08,487 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=248 (bloomFilter=true), to=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/.tmp/A/d040c025d62e44988c9596e31e8b49e2 2024-11-22T03:41:08,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/.tmp/B/864d7ce5710243e0a9ab7eb1306c5507 is 50, key is test_row_0/B:col10/1732246866728/Put/seqid=0 2024-11-22T03:41:08,504 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742080_1256 (size=12151) 2024-11-22T03:41:08,541 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:08,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39242 deadline: 1732246928539, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:08,541 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:08,541 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:08,541 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:08,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39154 deadline: 1732246928539, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:08,541 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:41:08,541 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:08,541 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:08,541 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:41:08,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-22T03:41:08,850 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:08,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39242 deadline: 1732246928848, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:08,850 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:08,850 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:08,850 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:08,850 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:41:08,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39154 deadline: 1732246928848, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:08,851 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:08,851 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:08,851 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:41:08,905 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=248 (bloomFilter=true), to=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/.tmp/B/864d7ce5710243e0a9ab7eb1306c5507 2024-11-22T03:41:08,911 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/.tmp/C/6dd9e1915ad143bfa36d8a1d194e99e7 is 50, key is test_row_0/C:col10/1732246866728/Put/seqid=0 2024-11-22T03:41:08,915 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742081_1257 (size=12151) 2024-11-22T03:41:08,930 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:08,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39226 deadline: 1732246928928, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:08,931 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to c85114ed5096,42545,1732246786482 for 'test_row_1' in f0af25cc84372b1efb01f18522e2d562 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2147 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-22T03:41:08,931 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:08,931 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:08,931 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:41:09,316 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=248 (bloomFilter=true), to=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/.tmp/C/6dd9e1915ad143bfa36d8a1d194e99e7 2024-11-22T03:41:09,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/.tmp/A/d040c025d62e44988c9596e31e8b49e2 as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/A/d040c025d62e44988c9596e31e8b49e2 2024-11-22T03:41:09,323 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/A/d040c025d62e44988c9596e31e8b49e2, entries=150, sequenceid=248, filesize=11.9 K 2024-11-22T03:41:09,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/.tmp/B/864d7ce5710243e0a9ab7eb1306c5507 as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/B/864d7ce5710243e0a9ab7eb1306c5507 2024-11-22T03:41:09,328 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/B/864d7ce5710243e0a9ab7eb1306c5507, entries=150, sequenceid=248, filesize=11.9 K 2024-11-22T03:41:09,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/.tmp/C/6dd9e1915ad143bfa36d8a1d194e99e7 as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/C/6dd9e1915ad143bfa36d8a1d194e99e7 2024-11-22T03:41:09,332 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/C/6dd9e1915ad143bfa36d8a1d194e99e7, entries=150, sequenceid=248, filesize=11.9 K 2024-11-22T03:41:09,332 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for f0af25cc84372b1efb01f18522e2d562 in 1257ms, sequenceid=248, compaction requested=false 2024-11-22T03:41:09,333 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2603): Flush status journal for f0af25cc84372b1efb01f18522e2d562: 2024-11-22T03:41:09,333 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562. 2024-11-22T03:41:09,333 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=75 2024-11-22T03:41:09,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42627 {}] master.HMaster(4169): Remote procedure done, pid=75 2024-11-22T03:41:09,336 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=75, resume processing ppid=74 2024-11-22T03:41:09,336 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=75, ppid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.6330 sec 2024-11-22T03:41:09,337 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees in 2.6370 sec 2024-11-22T03:41:09,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(8855): Flush requested on f0af25cc84372b1efb01f18522e2d562 2024-11-22T03:41:09,361 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-22T03:41:09,361 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f0af25cc84372b1efb01f18522e2d562 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-22T03:41:09,368 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/.tmp/A/4e43a5bc1c7e44fdbe5219c2fae74724 is 50, key is test_row_0/A:col10/1732246869360/Put/seqid=0 2024-11-22T03:41:09,373 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742082_1258 (size=14691) 2024-11-22T03:41:09,412 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:09,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39242 deadline: 1732246929409, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:09,412 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:09,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39154 deadline: 1732246929410, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:09,413 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:09,413 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:09,413 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:09,413 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:09,413 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:41:09,413 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:41:09,521 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:09,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39154 deadline: 1732246929519, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:09,522 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:09,522 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:09,522 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:09,522 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:41:09,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39242 deadline: 1732246929519, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:09,522 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:09,522 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:09,522 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:41:09,730 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:09,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39154 deadline: 1732246929729, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:09,730 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:09,730 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:09,730 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:09,730 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:41:09,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39242 deadline: 1732246929729, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:09,731 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:09,731 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:09,731 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:41:09,773 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=262 (bloomFilter=true), to=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/.tmp/A/4e43a5bc1c7e44fdbe5219c2fae74724 2024-11-22T03:41:09,780 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/.tmp/B/e6d819e644a1413a87b34ea930ddb363 is 50, key is test_row_0/B:col10/1732246869360/Put/seqid=0 2024-11-22T03:41:09,787 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742083_1259 (size=12251) 2024-11-22T03:41:10,041 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:10,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39154 deadline: 1732246930038, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:10,041 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:10,041 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:10,041 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:10,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39242 deadline: 1732246930038, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:10,041 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:41:10,041 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:10,041 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:10,042 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:41:10,188 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=262 (bloomFilter=true), to=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/.tmp/B/e6d819e644a1413a87b34ea930ddb363 2024-11-22T03:41:10,194 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/.tmp/C/a0ab28bd3c5049b780a0c0fb25b0c2c6 is 50, key is test_row_0/C:col10/1732246869360/Put/seqid=0 2024-11-22T03:41:10,197 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742084_1260 (size=12251) 2024-11-22T03:41:10,551 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:10,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39154 deadline: 1732246930548, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:10,551 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:10,551 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:10,551 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:10,551 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:41:10,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39242 deadline: 1732246930548, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:10,551 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:10,552 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:10,552 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:41:10,598 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=262 (bloomFilter=true), to=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/.tmp/C/a0ab28bd3c5049b780a0c0fb25b0c2c6 2024-11-22T03:41:10,602 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/.tmp/A/4e43a5bc1c7e44fdbe5219c2fae74724 as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/A/4e43a5bc1c7e44fdbe5219c2fae74724 2024-11-22T03:41:10,606 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/A/4e43a5bc1c7e44fdbe5219c2fae74724, entries=200, sequenceid=262, filesize=14.3 K 2024-11-22T03:41:10,607 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/.tmp/B/e6d819e644a1413a87b34ea930ddb363 as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/B/e6d819e644a1413a87b34ea930ddb363 2024-11-22T03:41:10,610 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/B/e6d819e644a1413a87b34ea930ddb363, entries=150, sequenceid=262, filesize=12.0 K 2024-11-22T03:41:10,611 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/.tmp/C/a0ab28bd3c5049b780a0c0fb25b0c2c6 as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/C/a0ab28bd3c5049b780a0c0fb25b0c2c6 2024-11-22T03:41:10,615 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/C/a0ab28bd3c5049b780a0c0fb25b0c2c6, entries=150, sequenceid=262, filesize=12.0 K 2024-11-22T03:41:10,616 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=140.89 KB/144270 for f0af25cc84372b1efb01f18522e2d562 in 1255ms, sequenceid=262, compaction requested=true 2024-11-22T03:41:10,616 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f0af25cc84372b1efb01f18522e2d562: 2024-11-22T03:41:10,616 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f0af25cc84372b1efb01f18522e2d562:A, priority=-2147483648, current under compaction store size is 1 2024-11-22T03:41:10,616 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T03:41:10,616 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T03:41:10,616 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f0af25cc84372b1efb01f18522e2d562:B, priority=-2147483648, current under compaction store size is 2 2024-11-22T03:41:10,616 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T03:41:10,616 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T03:41:10,616 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f0af25cc84372b1efb01f18522e2d562:C, priority=-2147483648, current under compaction store size is 3 2024-11-22T03:41:10,616 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T03:41:10,617 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39505 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T03:41:10,617 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37065 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T03:41:10,617 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.HStore(1541): f0af25cc84372b1efb01f18522e2d562/A is initiating minor compaction (all files) 2024-11-22T03:41:10,617 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HStore(1541): f0af25cc84372b1efb01f18522e2d562/B is initiating minor compaction (all files) 2024-11-22T03:41:10,617 INFO [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f0af25cc84372b1efb01f18522e2d562/A in TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562. 2024-11-22T03:41:10,617 INFO [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f0af25cc84372b1efb01f18522e2d562/B in TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562. 2024-11-22T03:41:10,617 INFO [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/A/d2bdfe5f6b534c8c9482bf20d51afc16, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/A/d040c025d62e44988c9596e31e8b49e2, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/A/4e43a5bc1c7e44fdbe5219c2fae74724] into tmpdir=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/.tmp, totalSize=38.6 K 2024-11-22T03:41:10,617 INFO [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/B/9d3c752c288140f99b9a2e93e576d3cd, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/B/864d7ce5710243e0a9ab7eb1306c5507, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/B/e6d819e644a1413a87b34ea930ddb363] into tmpdir=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/.tmp, totalSize=36.2 K 2024-11-22T03:41:10,618 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] compactions.Compactor(225): Compacting 9d3c752c288140f99b9a2e93e576d3cd, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=221, earliestPutTs=1732246866693 2024-11-22T03:41:10,618 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] compactions.Compactor(225): Compacting d2bdfe5f6b534c8c9482bf20d51afc16, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=221, earliestPutTs=1732246866693 2024-11-22T03:41:10,618 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] compactions.Compactor(225): Compacting 864d7ce5710243e0a9ab7eb1306c5507, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=248, earliestPutTs=1732246866728 2024-11-22T03:41:10,618 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] compactions.Compactor(225): Compacting d040c025d62e44988c9596e31e8b49e2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=248, earliestPutTs=1732246866728 2024-11-22T03:41:10,619 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] compactions.Compactor(225): Compacting e6d819e644a1413a87b34ea930ddb363, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=262, earliestPutTs=1732246868207 2024-11-22T03:41:10,619 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4e43a5bc1c7e44fdbe5219c2fae74724, keycount=200, bloomtype=ROW, size=14.3 K, encoding=NONE, compression=NONE, seqNum=262, earliestPutTs=1732246868207 2024-11-22T03:41:10,626 INFO [RS:0;c85114ed5096:42545-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f0af25cc84372b1efb01f18522e2d562#B#compaction#225 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-22T03:41:10,626 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/.tmp/B/5c63f0f2f42f4820b9670c6ef1f2c10f is 50, key is test_row_0/B:col10/1732246869360/Put/seqid=0 2024-11-22T03:41:10,635 INFO [RS:0;c85114ed5096:42545-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f0af25cc84372b1efb01f18522e2d562#A#compaction#226 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-22T03:41:10,636 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/.tmp/A/fb9c8e9a99834bdaa04449c7fe661802 is 50, key is test_row_0/A:col10/1732246869360/Put/seqid=0 2024-11-22T03:41:10,638 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742085_1261 (size=12865) 2024-11-22T03:41:10,640 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742086_1262 (size=12865) 2024-11-22T03:41:10,646 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/.tmp/A/fb9c8e9a99834bdaa04449c7fe661802 as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/A/fb9c8e9a99834bdaa04449c7fe661802 2024-11-22T03:41:10,646 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/.tmp/B/5c63f0f2f42f4820b9670c6ef1f2c10f as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/B/5c63f0f2f42f4820b9670c6ef1f2c10f 2024-11-22T03:41:10,654 INFO [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f0af25cc84372b1efb01f18522e2d562/B of f0af25cc84372b1efb01f18522e2d562 into 5c63f0f2f42f4820b9670c6ef1f2c10f(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T03:41:10,654 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f0af25cc84372b1efb01f18522e2d562: 2024-11-22T03:41:10,654 INFO [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., storeName=f0af25cc84372b1efb01f18522e2d562/B, priority=13, startTime=1732246870616; duration=0sec 2024-11-22T03:41:10,654 INFO [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f0af25cc84372b1efb01f18522e2d562/A of f0af25cc84372b1efb01f18522e2d562 into fb9c8e9a99834bdaa04449c7fe661802(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T03:41:10,654 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f0af25cc84372b1efb01f18522e2d562: 2024-11-22T03:41:10,654 INFO [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., storeName=f0af25cc84372b1efb01f18522e2d562/A, priority=13, startTime=1732246870616; duration=0sec 2024-11-22T03:41:10,654 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T03:41:10,654 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f0af25cc84372b1efb01f18522e2d562:B 2024-11-22T03:41:10,654 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T03:41:10,654 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f0af25cc84372b1efb01f18522e2d562:A 2024-11-22T03:41:10,654 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T03:41:10,655 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37065 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T03:41:10,655 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HStore(1541): f0af25cc84372b1efb01f18522e2d562/C is initiating minor compaction (all files) 2024-11-22T03:41:10,655 INFO [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f0af25cc84372b1efb01f18522e2d562/C in TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562. 2024-11-22T03:41:10,655 INFO [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/C/8cc3162719944edea3667433d5827b83, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/C/6dd9e1915ad143bfa36d8a1d194e99e7, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/C/a0ab28bd3c5049b780a0c0fb25b0c2c6] into tmpdir=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/.tmp, totalSize=36.2 K 2024-11-22T03:41:10,656 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] compactions.Compactor(225): Compacting 8cc3162719944edea3667433d5827b83, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=221, earliestPutTs=1732246866693 2024-11-22T03:41:10,656 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] compactions.Compactor(225): Compacting 6dd9e1915ad143bfa36d8a1d194e99e7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=248, earliestPutTs=1732246866728 2024-11-22T03:41:10,656 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] compactions.Compactor(225): Compacting a0ab28bd3c5049b780a0c0fb25b0c2c6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=262, earliestPutTs=1732246868207 2024-11-22T03:41:10,663 INFO [RS:0;c85114ed5096:42545-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f0af25cc84372b1efb01f18522e2d562#C#compaction#227 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-22T03:41:10,663 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/.tmp/C/223f6c0b772f476ea8383f7767b76aec is 50, key is test_row_0/C:col10/1732246869360/Put/seqid=0 2024-11-22T03:41:10,671 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742087_1263 (size=12865) 2024-11-22T03:41:10,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-22T03:41:10,869 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-22T03:41:10,869 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42627 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-22T03:41:10,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42627 {}] procedure2.ProcedureExecutor(1139): Stored pid=76, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees 2024-11-22T03:41:10,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-22T03:41:10,871 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=76, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-22T03:41:10,871 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=76, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-22T03:41:10,872 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=77, ppid=76, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-22T03:41:10,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(8855): Flush requested on f0af25cc84372b1efb01f18522e2d562 2024-11-22T03:41:10,953 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-22T03:41:10,953 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f0af25cc84372b1efb01f18522e2d562 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-22T03:41:10,958 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/.tmp/A/940e4b592a99498292b706626c494285 is 50, key is test_row_0/A:col10/1732246869404/Put/seqid=0 2024-11-22T03:41:10,963 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742088_1264 (size=12301) 2024-11-22T03:41:10,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-22T03:41:10,980 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:10,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39226 deadline: 1732246930978, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:10,980 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:10,980 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:10,980 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:41:11,023 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42545 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-22T03:41:11,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562. 2024-11-22T03:41:11,024 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562. as already flushing 2024-11-22T03:41:11,024 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562. 2024-11-22T03:41:11,024 ERROR [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => f0af25cc84372b1efb01f18522e2d562, NAME => 'TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:41:11,024 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => f0af25cc84372b1efb01f18522e2d562, NAME => 'TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:41:11,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42627 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f0af25cc84372b1efb01f18522e2d562, NAME => 'TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f0af25cc84372b1efb01f18522e2d562, NAME => 'TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:41:11,076 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/.tmp/C/223f6c0b772f476ea8383f7767b76aec as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/C/223f6c0b772f476ea8383f7767b76aec 2024-11-22T03:41:11,080 INFO [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f0af25cc84372b1efb01f18522e2d562/C of f0af25cc84372b1efb01f18522e2d562 into 223f6c0b772f476ea8383f7767b76aec(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T03:41:11,080 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f0af25cc84372b1efb01f18522e2d562: 2024-11-22T03:41:11,080 INFO [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., storeName=f0af25cc84372b1efb01f18522e2d562/C, priority=13, startTime=1732246870616; duration=0sec 2024-11-22T03:41:11,080 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T03:41:11,080 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f0af25cc84372b1efb01f18522e2d562:C 2024-11-22T03:41:11,091 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:11,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39226 deadline: 1732246931089, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:11,091 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:11,091 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:11,091 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:41:11,176 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42545 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-22T03:41:11,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562. 2024-11-22T03:41:11,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562. as already flushing 2024-11-22T03:41:11,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562. 2024-11-22T03:41:11,176 ERROR [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => f0af25cc84372b1efb01f18522e2d562, NAME => 'TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:41:11,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => f0af25cc84372b1efb01f18522e2d562, NAME => 'TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:41:11,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42627 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f0af25cc84372b1efb01f18522e2d562, NAME => 'TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f0af25cc84372b1efb01f18522e2d562, NAME => 'TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:41:11,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-22T03:41:11,300 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:11,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39226 deadline: 1732246931298, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:11,301 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:11,301 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:11,301 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:41:11,329 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42545 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-22T03:41:11,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562. 2024-11-22T03:41:11,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562. as already flushing 2024-11-22T03:41:11,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562. 2024-11-22T03:41:11,329 ERROR [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => f0af25cc84372b1efb01f18522e2d562, NAME => 'TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:41:11,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => f0af25cc84372b1efb01f18522e2d562, NAME => 'TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:41:11,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42627 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f0af25cc84372b1efb01f18522e2d562, NAME => 'TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f0af25cc84372b1efb01f18522e2d562, NAME => 'TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:41:11,363 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=289 (bloomFilter=true), to=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/.tmp/A/940e4b592a99498292b706626c494285 2024-11-22T03:41:11,369 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/.tmp/B/9f9f64fb1a364458ac050c2c2d400d9e is 50, key is test_row_0/B:col10/1732246869404/Put/seqid=0 2024-11-22T03:41:11,373 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742089_1265 (size=12301) 2024-11-22T03:41:11,482 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42545 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-22T03:41:11,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562. 2024-11-22T03:41:11,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562. as already flushing 2024-11-22T03:41:11,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562. 2024-11-22T03:41:11,482 ERROR [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => f0af25cc84372b1efb01f18522e2d562, NAME => 'TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:41:11,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => f0af25cc84372b1efb01f18522e2d562, NAME => 'TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:41:11,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42627 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f0af25cc84372b1efb01f18522e2d562, NAME => 'TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f0af25cc84372b1efb01f18522e2d562, NAME => 'TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:41:11,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-22T03:41:11,560 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:11,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39154 deadline: 1732246931559, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:11,560 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:11,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39242 deadline: 1732246931559, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:11,560 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to c85114ed5096,42545,1732246786482 for 'test_row_2' in f0af25cc84372b1efb01f18522e2d562 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2150 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-22T03:41:11,561 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:11,561 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:11,560 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to c85114ed5096,42545,1732246786482 for 'test_row_2' in f0af25cc84372b1efb01f18522e2d562 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2151 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-22T03:41:11,561 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:41:11,561 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:11,561 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:11,561 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:41:11,609 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:11,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39226 deadline: 1732246931608, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:11,610 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:11,610 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:11,610 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:41:11,633 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42545 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-22T03:41:11,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562. 2024-11-22T03:41:11,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562. as already flushing 2024-11-22T03:41:11,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562. 2024-11-22T03:41:11,634 ERROR [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => f0af25cc84372b1efb01f18522e2d562, NAME => 'TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:41:11,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => f0af25cc84372b1efb01f18522e2d562, NAME => 'TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:41:11,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42627 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f0af25cc84372b1efb01f18522e2d562, NAME => 'TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f0af25cc84372b1efb01f18522e2d562, NAME => 'TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:41:11,774 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=289 (bloomFilter=true), to=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/.tmp/B/9f9f64fb1a364458ac050c2c2d400d9e 2024-11-22T03:41:11,782 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/.tmp/C/26dd7574bcc841bca4de79647f1fe14a is 50, key is test_row_0/C:col10/1732246869404/Put/seqid=0 2024-11-22T03:41:11,784 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742090_1266 (size=12301) 2024-11-22T03:41:11,786 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42545 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-22T03:41:11,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562. 2024-11-22T03:41:11,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562. as already flushing 2024-11-22T03:41:11,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562. 2024-11-22T03:41:11,786 ERROR [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => f0af25cc84372b1efb01f18522e2d562, NAME => 'TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:41:11,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => f0af25cc84372b1efb01f18522e2d562, NAME => 'TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:41:11,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42627 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f0af25cc84372b1efb01f18522e2d562, NAME => 'TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f0af25cc84372b1efb01f18522e2d562, NAME => 'TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:41:11,938 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42545 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-22T03:41:11,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562. 2024-11-22T03:41:11,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562. as already flushing 2024-11-22T03:41:11,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562. 2024-11-22T03:41:11,939 ERROR [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => f0af25cc84372b1efb01f18522e2d562, NAME => 'TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:41:11,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => f0af25cc84372b1efb01f18522e2d562, NAME => 'TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:41:11,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42627 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f0af25cc84372b1efb01f18522e2d562, NAME => 'TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f0af25cc84372b1efb01f18522e2d562, NAME => 'TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:41:12,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-22T03:41:12,091 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42545 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-22T03:41:12,091 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562. 2024-11-22T03:41:12,091 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562. as already flushing 2024-11-22T03:41:12,091 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562. 2024-11-22T03:41:12,091 ERROR [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => f0af25cc84372b1efb01f18522e2d562, NAME => 'TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:41:12,091 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => f0af25cc84372b1efb01f18522e2d562, NAME => 'TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:41:12,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42627 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f0af25cc84372b1efb01f18522e2d562, NAME => 'TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f0af25cc84372b1efb01f18522e2d562, NAME => 'TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:41:12,122 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:12,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39226 deadline: 1732246932119, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:12,122 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:12,122 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:12,122 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:41:12,185 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=289 (bloomFilter=true), to=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/.tmp/C/26dd7574bcc841bca4de79647f1fe14a 2024-11-22T03:41:12,189 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/.tmp/A/940e4b592a99498292b706626c494285 as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/A/940e4b592a99498292b706626c494285 2024-11-22T03:41:12,192 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/A/940e4b592a99498292b706626c494285, entries=150, sequenceid=289, filesize=12.0 K 2024-11-22T03:41:12,192 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/.tmp/B/9f9f64fb1a364458ac050c2c2d400d9e as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/B/9f9f64fb1a364458ac050c2c2d400d9e 2024-11-22T03:41:12,196 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/B/9f9f64fb1a364458ac050c2c2d400d9e, entries=150, sequenceid=289, filesize=12.0 K 2024-11-22T03:41:12,196 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/.tmp/C/26dd7574bcc841bca4de79647f1fe14a as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/C/26dd7574bcc841bca4de79647f1fe14a 2024-11-22T03:41:12,200 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/C/26dd7574bcc841bca4de79647f1fe14a, entries=150, sequenceid=289, filesize=12.0 K 2024-11-22T03:41:12,200 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for f0af25cc84372b1efb01f18522e2d562 in 1247ms, sequenceid=289, compaction requested=false 2024-11-22T03:41:12,200 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f0af25cc84372b1efb01f18522e2d562: 2024-11-22T03:41:12,244 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42545 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-22T03:41:12,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562. 2024-11-22T03:41:12,244 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2902): Flushing f0af25cc84372b1efb01f18522e2d562 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-22T03:41:12,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/.tmp/A/2f88255cd7a34a67a114fcb3a3b99e88 is 50, key is test_row_0/A:col10/1732246870963/Put/seqid=0 2024-11-22T03:41:12,252 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742091_1267 (size=9857) 2024-11-22T03:41:12,653 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=301 (bloomFilter=true), to=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/.tmp/A/2f88255cd7a34a67a114fcb3a3b99e88 2024-11-22T03:41:12,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/.tmp/B/8ac49ea535504f1e88601386d3c01011 is 50, key is test_row_0/B:col10/1732246870963/Put/seqid=0 2024-11-22T03:41:12,663 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742092_1268 (size=9857) 2024-11-22T03:41:12,663 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=301 (bloomFilter=true), to=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/.tmp/B/8ac49ea535504f1e88601386d3c01011 2024-11-22T03:41:12,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/.tmp/C/3b89a8cb841f450ba9c2e9fbb1a08f61 is 50, key is test_row_0/C:col10/1732246870963/Put/seqid=0 2024-11-22T03:41:12,690 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742093_1269 (size=9857) 2024-11-22T03:41:13,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-22T03:41:13,091 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=301 (bloomFilter=true), to=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/.tmp/C/3b89a8cb841f450ba9c2e9fbb1a08f61 2024-11-22T03:41:13,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/.tmp/A/2f88255cd7a34a67a114fcb3a3b99e88 as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/A/2f88255cd7a34a67a114fcb3a3b99e88 2024-11-22T03:41:13,099 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/A/2f88255cd7a34a67a114fcb3a3b99e88, entries=100, sequenceid=301, filesize=9.6 K 2024-11-22T03:41:13,099 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/.tmp/B/8ac49ea535504f1e88601386d3c01011 as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/B/8ac49ea535504f1e88601386d3c01011 2024-11-22T03:41:13,106 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/B/8ac49ea535504f1e88601386d3c01011, entries=100, sequenceid=301, filesize=9.6 K 2024-11-22T03:41:13,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/.tmp/C/3b89a8cb841f450ba9c2e9fbb1a08f61 as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/C/3b89a8cb841f450ba9c2e9fbb1a08f61 2024-11-22T03:41:13,111 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/C/3b89a8cb841f450ba9c2e9fbb1a08f61, entries=100, sequenceid=301, filesize=9.6 K 2024-11-22T03:41:13,112 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=0 B/0 for f0af25cc84372b1efb01f18522e2d562 in 868ms, sequenceid=301, compaction requested=true 2024-11-22T03:41:13,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2603): Flush status journal for f0af25cc84372b1efb01f18522e2d562: 2024-11-22T03:41:13,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562. 2024-11-22T03:41:13,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=77 2024-11-22T03:41:13,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42627 {}] master.HMaster(4169): Remote procedure done, pid=77 2024-11-22T03:41:13,116 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=77, resume processing ppid=76 2024-11-22T03:41:13,116 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=77, ppid=76, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2430 sec 2024-11-22T03:41:13,118 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=76, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees in 2.2470 sec 2024-11-22T03:41:13,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(8855): Flush requested on f0af25cc84372b1efb01f18522e2d562 2024-11-22T03:41:13,149 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-22T03:41:13,149 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f0af25cc84372b1efb01f18522e2d562 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-22T03:41:13,153 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/.tmp/A/0744fbb81b434e749b597f834b1038bc is 50, key is test_row_0/A:col10/1732246873148/Put/seqid=0 2024-11-22T03:41:13,160 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742094_1270 (size=14741) 2024-11-22T03:41:13,160 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=312 (bloomFilter=true), to=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/.tmp/A/0744fbb81b434e749b597f834b1038bc 2024-11-22T03:41:13,166 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/.tmp/B/b2a4f05e734b435daef1ad0203c1e978 is 50, key is test_row_0/B:col10/1732246873148/Put/seqid=0 2024-11-22T03:41:13,170 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742095_1271 (size=12301) 2024-11-22T03:41:13,241 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:13,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 181 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39226 deadline: 1732246933238, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:13,242 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:13,242 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:13,242 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:41:13,352 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:13,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 182 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39226 deadline: 1732246933349, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:13,352 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:13,352 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:13,352 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:41:13,561 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:13,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 183 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39226 deadline: 1732246933559, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:13,561 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:13,562 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:13,562 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:41:13,571 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=312 (bloomFilter=true), to=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/.tmp/B/b2a4f05e734b435daef1ad0203c1e978 2024-11-22T03:41:13,577 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/.tmp/C/16ea1d93a16e4a5a8e8f6e7eb191e311 is 50, key is test_row_0/C:col10/1732246873148/Put/seqid=0 2024-11-22T03:41:13,582 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:13,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39154 deadline: 1732246933579, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:13,582 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:13,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39242 deadline: 1732246933579, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:13,583 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to c85114ed5096,42545,1732246786482 for 'test_row_2' in f0af25cc84372b1efb01f18522e2d562 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4172 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-22T03:41:13,583 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:13,583 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:13,583 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:41:13,583 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to c85114ed5096,42545,1732246786482 for 'test_row_2' in f0af25cc84372b1efb01f18522e2d562 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4173 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-22T03:41:13,583 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:13,583 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:13,583 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:41:13,588 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742096_1272 (size=12301) 2024-11-22T03:41:13,871 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:13,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 184 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39226 deadline: 1732246933868, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:13,871 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:13,871 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:13,871 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:41:13,989 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=312 (bloomFilter=true), to=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/.tmp/C/16ea1d93a16e4a5a8e8f6e7eb191e311 2024-11-22T03:41:13,993 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/.tmp/A/0744fbb81b434e749b597f834b1038bc as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/A/0744fbb81b434e749b597f834b1038bc 2024-11-22T03:41:13,996 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/A/0744fbb81b434e749b597f834b1038bc, entries=200, sequenceid=312, filesize=14.4 K 2024-11-22T03:41:13,997 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/.tmp/B/b2a4f05e734b435daef1ad0203c1e978 as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/B/b2a4f05e734b435daef1ad0203c1e978 2024-11-22T03:41:14,000 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/B/b2a4f05e734b435daef1ad0203c1e978, entries=150, sequenceid=312, filesize=12.0 K 2024-11-22T03:41:14,001 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/.tmp/C/16ea1d93a16e4a5a8e8f6e7eb191e311 as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/C/16ea1d93a16e4a5a8e8f6e7eb191e311 2024-11-22T03:41:14,005 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/C/16ea1d93a16e4a5a8e8f6e7eb191e311, entries=150, sequenceid=312, filesize=12.0 K 2024-11-22T03:41:14,006 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for f0af25cc84372b1efb01f18522e2d562 in 857ms, sequenceid=312, compaction requested=true 2024-11-22T03:41:14,006 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f0af25cc84372b1efb01f18522e2d562: 2024-11-22T03:41:14,006 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f0af25cc84372b1efb01f18522e2d562:A, priority=-2147483648, current under compaction store size is 1 2024-11-22T03:41:14,006 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T03:41:14,006 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f0af25cc84372b1efb01f18522e2d562:B, priority=-2147483648, current under compaction store size is 2 2024-11-22T03:41:14,006 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-22T03:41:14,006 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T03:41:14,006 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-22T03:41:14,006 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f0af25cc84372b1efb01f18522e2d562:C, priority=-2147483648, current under compaction store size is 3 2024-11-22T03:41:14,006 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T03:41:14,007 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 47324 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-22T03:41:14,007 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49764 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-22T03:41:14,007 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HStore(1541): f0af25cc84372b1efb01f18522e2d562/B is initiating minor compaction (all files) 2024-11-22T03:41:14,007 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.HStore(1541): f0af25cc84372b1efb01f18522e2d562/A is initiating minor compaction (all files) 2024-11-22T03:41:14,007 INFO [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f0af25cc84372b1efb01f18522e2d562/B in TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562. 2024-11-22T03:41:14,007 INFO [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f0af25cc84372b1efb01f18522e2d562/A in TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562. 2024-11-22T03:41:14,007 INFO [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/A/fb9c8e9a99834bdaa04449c7fe661802, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/A/940e4b592a99498292b706626c494285, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/A/2f88255cd7a34a67a114fcb3a3b99e88, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/A/0744fbb81b434e749b597f834b1038bc] into tmpdir=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/.tmp, totalSize=48.6 K 2024-11-22T03:41:14,007 INFO [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/B/5c63f0f2f42f4820b9670c6ef1f2c10f, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/B/9f9f64fb1a364458ac050c2c2d400d9e, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/B/8ac49ea535504f1e88601386d3c01011, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/B/b2a4f05e734b435daef1ad0203c1e978] into tmpdir=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/.tmp, totalSize=46.2 K 2024-11-22T03:41:14,007 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] compactions.Compactor(225): Compacting fb9c8e9a99834bdaa04449c7fe661802, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=262, earliestPutTs=1732246868207 2024-11-22T03:41:14,007 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] compactions.Compactor(225): Compacting 5c63f0f2f42f4820b9670c6ef1f2c10f, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=262, earliestPutTs=1732246868207 2024-11-22T03:41:14,008 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] compactions.Compactor(225): Compacting 940e4b592a99498292b706626c494285, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=289, earliestPutTs=1732246869394 2024-11-22T03:41:14,008 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] compactions.Compactor(225): Compacting 9f9f64fb1a364458ac050c2c2d400d9e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=289, earliestPutTs=1732246869394 2024-11-22T03:41:14,008 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2f88255cd7a34a67a114fcb3a3b99e88, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=301, earliestPutTs=1732246870963 2024-11-22T03:41:14,008 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] compactions.Compactor(225): Compacting 8ac49ea535504f1e88601386d3c01011, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=301, earliestPutTs=1732246870963 2024-11-22T03:41:14,008 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0744fbb81b434e749b597f834b1038bc, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=312, earliestPutTs=1732246873128 2024-11-22T03:41:14,008 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] compactions.Compactor(225): Compacting b2a4f05e734b435daef1ad0203c1e978, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=312, earliestPutTs=1732246873128 2024-11-22T03:41:14,017 INFO [RS:0;c85114ed5096:42545-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f0af25cc84372b1efb01f18522e2d562#A#compaction#237 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-22T03:41:14,018 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/.tmp/A/74716376f38144ea96f895cc1c170fab is 50, key is test_row_0/A:col10/1732246873148/Put/seqid=0 2024-11-22T03:41:14,027 INFO [RS:0;c85114ed5096:42545-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f0af25cc84372b1efb01f18522e2d562#B#compaction#238 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-22T03:41:14,028 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/.tmp/B/87390e5b4d20420b81efe8d95e7447ff is 50, key is test_row_0/B:col10/1732246873148/Put/seqid=0 2024-11-22T03:41:14,030 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742097_1273 (size=13051) 2024-11-22T03:41:14,032 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742098_1274 (size=13051) 2024-11-22T03:41:14,037 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/.tmp/B/87390e5b4d20420b81efe8d95e7447ff as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/B/87390e5b4d20420b81efe8d95e7447ff 2024-11-22T03:41:14,042 INFO [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in f0af25cc84372b1efb01f18522e2d562/B of f0af25cc84372b1efb01f18522e2d562 into 87390e5b4d20420b81efe8d95e7447ff(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T03:41:14,042 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f0af25cc84372b1efb01f18522e2d562: 2024-11-22T03:41:14,042 INFO [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., storeName=f0af25cc84372b1efb01f18522e2d562/B, priority=12, startTime=1732246874006; duration=0sec 2024-11-22T03:41:14,042 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T03:41:14,042 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f0af25cc84372b1efb01f18522e2d562:B 2024-11-22T03:41:14,042 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-22T03:41:14,043 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 47324 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-22T03:41:14,043 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HStore(1541): f0af25cc84372b1efb01f18522e2d562/C is initiating minor compaction (all files) 2024-11-22T03:41:14,044 INFO [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f0af25cc84372b1efb01f18522e2d562/C in TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562. 2024-11-22T03:41:14,044 INFO [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/C/223f6c0b772f476ea8383f7767b76aec, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/C/26dd7574bcc841bca4de79647f1fe14a, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/C/3b89a8cb841f450ba9c2e9fbb1a08f61, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/C/16ea1d93a16e4a5a8e8f6e7eb191e311] into tmpdir=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/.tmp, totalSize=46.2 K 2024-11-22T03:41:14,044 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] compactions.Compactor(225): Compacting 223f6c0b772f476ea8383f7767b76aec, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=262, earliestPutTs=1732246868207 2024-11-22T03:41:14,044 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] compactions.Compactor(225): Compacting 26dd7574bcc841bca4de79647f1fe14a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=289, earliestPutTs=1732246869394 2024-11-22T03:41:14,045 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] compactions.Compactor(225): Compacting 3b89a8cb841f450ba9c2e9fbb1a08f61, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=301, earliestPutTs=1732246870963 2024-11-22T03:41:14,045 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] compactions.Compactor(225): Compacting 16ea1d93a16e4a5a8e8f6e7eb191e311, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=312, earliestPutTs=1732246873128 2024-11-22T03:41:14,055 INFO [RS:0;c85114ed5096:42545-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f0af25cc84372b1efb01f18522e2d562#C#compaction#239 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-22T03:41:14,056 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/.tmp/C/9c9c7a7d54794d66872a2d1dfa9d159e is 50, key is test_row_0/C:col10/1732246873148/Put/seqid=0 2024-11-22T03:41:14,059 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742099_1275 (size=13051) 2024-11-22T03:41:14,066 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/.tmp/C/9c9c7a7d54794d66872a2d1dfa9d159e as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/C/9c9c7a7d54794d66872a2d1dfa9d159e 2024-11-22T03:41:14,071 INFO [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in f0af25cc84372b1efb01f18522e2d562/C of f0af25cc84372b1efb01f18522e2d562 into 9c9c7a7d54794d66872a2d1dfa9d159e(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T03:41:14,071 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f0af25cc84372b1efb01f18522e2d562: 2024-11-22T03:41:14,071 INFO [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., storeName=f0af25cc84372b1efb01f18522e2d562/C, priority=12, startTime=1732246874006; duration=0sec 2024-11-22T03:41:14,071 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T03:41:14,071 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f0af25cc84372b1efb01f18522e2d562:C 2024-11-22T03:41:14,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(8855): Flush requested on f0af25cc84372b1efb01f18522e2d562 2024-11-22T03:41:14,380 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-22T03:41:14,380 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f0af25cc84372b1efb01f18522e2d562 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-22T03:41:14,385 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/.tmp/A/8c92da645d5946559eff862a0b4ca3ca is 50, key is test_row_0/A:col10/1732246873237/Put/seqid=0 2024-11-22T03:41:14,390 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742100_1276 (size=12301) 2024-11-22T03:41:14,391 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=340 (bloomFilter=true), to=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/.tmp/A/8c92da645d5946559eff862a0b4ca3ca 2024-11-22T03:41:14,401 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/.tmp/B/9c10da7702884f8c9ab2f501d6e370e6 is 50, key is test_row_0/B:col10/1732246873237/Put/seqid=0 2024-11-22T03:41:14,402 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:14,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 193 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39226 deadline: 1732246934400, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:14,403 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:14,403 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:14,403 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:41:14,406 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742101_1277 (size=12301) 2024-11-22T03:41:14,435 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/.tmp/A/74716376f38144ea96f895cc1c170fab as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/A/74716376f38144ea96f895cc1c170fab 2024-11-22T03:41:14,440 INFO [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in f0af25cc84372b1efb01f18522e2d562/A of f0af25cc84372b1efb01f18522e2d562 into 74716376f38144ea96f895cc1c170fab(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T03:41:14,440 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f0af25cc84372b1efb01f18522e2d562: 2024-11-22T03:41:14,440 INFO [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., storeName=f0af25cc84372b1efb01f18522e2d562/A, priority=12, startTime=1732246874006; duration=0sec 2024-11-22T03:41:14,440 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T03:41:14,440 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f0af25cc84372b1efb01f18522e2d562:A 2024-11-22T03:41:14,510 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:14,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 194 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39226 deadline: 1732246934509, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:14,511 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:14,511 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:14,511 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:41:14,720 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:14,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 195 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39226 deadline: 1732246934719, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:14,720 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:14,720 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:14,720 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:41:14,807 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=340 (bloomFilter=true), to=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/.tmp/B/9c10da7702884f8c9ab2f501d6e370e6 2024-11-22T03:41:14,814 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/.tmp/C/1746a321d3c2404bba9594cb37cdf442 is 50, key is test_row_0/C:col10/1732246873237/Put/seqid=0 2024-11-22T03:41:14,821 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742102_1278 (size=12301) 2024-11-22T03:41:14,823 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-22T03:41:15,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-22T03:41:15,028 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-22T03:41:15,029 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42627 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-22T03:41:15,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42627 {}] procedure2.ProcedureExecutor(1139): Stored pid=78, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees 2024-11-22T03:41:15,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-22T03:41:15,030 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=78, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-22T03:41:15,030 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:15,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 196 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39226 deadline: 1732246935028, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:15,031 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=78, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-22T03:41:15,031 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:15,031 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:15,031 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=79, ppid=78, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-22T03:41:15,031 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:41:15,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-22T03:41:15,182 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42545 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-22T03:41:15,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562. 2024-11-22T03:41:15,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562. as already flushing 2024-11-22T03:41:15,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562. 2024-11-22T03:41:15,183 ERROR [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => f0af25cc84372b1efb01f18522e2d562, NAME => 'TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:41:15,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => f0af25cc84372b1efb01f18522e2d562, NAME => 'TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:41:15,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42627 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f0af25cc84372b1efb01f18522e2d562, NAME => 'TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f0af25cc84372b1efb01f18522e2d562, NAME => 'TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:41:15,221 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=340 (bloomFilter=true), to=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/.tmp/C/1746a321d3c2404bba9594cb37cdf442 2024-11-22T03:41:15,225 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/.tmp/A/8c92da645d5946559eff862a0b4ca3ca as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/A/8c92da645d5946559eff862a0b4ca3ca 2024-11-22T03:41:15,228 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/A/8c92da645d5946559eff862a0b4ca3ca, entries=150, sequenceid=340, filesize=12.0 K 2024-11-22T03:41:15,229 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/.tmp/B/9c10da7702884f8c9ab2f501d6e370e6 as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/B/9c10da7702884f8c9ab2f501d6e370e6 2024-11-22T03:41:15,233 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/B/9c10da7702884f8c9ab2f501d6e370e6, entries=150, sequenceid=340, filesize=12.0 K 2024-11-22T03:41:15,234 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/.tmp/C/1746a321d3c2404bba9594cb37cdf442 as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/C/1746a321d3c2404bba9594cb37cdf442 2024-11-22T03:41:15,237 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/C/1746a321d3c2404bba9594cb37cdf442, entries=150, sequenceid=340, filesize=12.0 K 2024-11-22T03:41:15,238 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for f0af25cc84372b1efb01f18522e2d562 in 858ms, sequenceid=340, compaction requested=false 2024-11-22T03:41:15,238 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f0af25cc84372b1efb01f18522e2d562: 2024-11-22T03:41:15,335 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42545 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-22T03:41:15,336 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562. 2024-11-22T03:41:15,336 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2902): Flushing f0af25cc84372b1efb01f18522e2d562 3/3 column families, dataSize=46.96 KB heapSize=123.80 KB 2024-11-22T03:41:15,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/.tmp/A/fb33ac7c77f640a397daa70d4d471fc4 is 50, key is test_row_0/A:col10/1732246874394/Put/seqid=0 2024-11-22T03:41:15,343 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742103_1279 (size=12301) 2024-11-22T03:41:15,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-22T03:41:15,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(8855): Flush requested on f0af25cc84372b1efb01f18522e2d562 2024-11-22T03:41:15,540 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562. as already flushing 2024-11-22T03:41:15,633 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:15,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 220 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39226 deadline: 1732246935629, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:15,633 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:15,633 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:15,633 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:41:15,655 INFO [Thread-1000 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1000. 2024-11-22T03:41:15,655 DEBUG [Thread-1000 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-22T03:41:15,655 DEBUG [Thread-1000 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T03:41:15,655 DEBUG [Thread-1000 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T03:41:15,655 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-22T03:41:15,657 INFO [Thread-996 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-996. 2024-11-22T03:41:15,657 DEBUG [Thread-996 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-22T03:41:15,657 DEBUG [Thread-996 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T03:41:15,657 DEBUG [Thread-996 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T03:41:15,657 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-22T03:41:15,657 INFO [Thread-998 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-998. 2024-11-22T03:41:15,657 DEBUG [Thread-998 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-22T03:41:15,657 DEBUG [Thread-998 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T03:41:15,658 DEBUG [Thread-998 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T03:41:15,658 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-22T03:41:15,658 INFO [Thread-1002 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1002. 2024-11-22T03:41:15,658 DEBUG [Thread-1002 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-22T03:41:15,658 DEBUG [Thread-1002 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T03:41:15,658 DEBUG [Thread-1002 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T03:41:15,658 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-22T03:41:15,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-22T03:41:15,660 INFO [Thread-1004 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1004. 2024-11-22T03:41:15,660 DEBUG [Thread-1004 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-22T03:41:15,660 DEBUG [Thread-1004 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T03:41:15,660 DEBUG [Thread-1004 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T03:41:15,660 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-22T03:41:15,739 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:15,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 221 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39226 deadline: 1732246935738, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:15,740 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:15,740 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:15,740 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:41:15,745 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=351 (bloomFilter=true), to=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/.tmp/A/fb33ac7c77f640a397daa70d4d471fc4 2024-11-22T03:41:15,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/.tmp/B/a1fded9ab59e445fa0393c9708538935 is 50, key is test_row_0/B:col10/1732246874394/Put/seqid=0 2024-11-22T03:41:15,760 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742104_1280 (size=12301) 2024-11-22T03:41:15,949 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:15,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 222 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39226 deadline: 1732246935949, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:15,950 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:15,950 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:15,950 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:41:16,161 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=351 (bloomFilter=true), to=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/.tmp/B/a1fded9ab59e445fa0393c9708538935 2024-11-22T03:41:16,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-22T03:41:16,173 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/.tmp/C/2d79847d3a1e48af991987147964ed29 is 50, key is test_row_0/C:col10/1732246874394/Put/seqid=0 2024-11-22T03:41:16,176 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742105_1281 (size=12301) 2024-11-22T03:41:16,259 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:16,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 223 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39226 deadline: 1732246936258, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:16,259 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:16,260 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:16,260 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:41:16,579 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=351 (bloomFilter=true), to=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/.tmp/C/2d79847d3a1e48af991987147964ed29 2024-11-22T03:41:16,591 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/.tmp/A/fb33ac7c77f640a397daa70d4d471fc4 as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/A/fb33ac7c77f640a397daa70d4d471fc4 2024-11-22T03:41:16,595 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/A/fb33ac7c77f640a397daa70d4d471fc4, entries=150, sequenceid=351, filesize=12.0 K 2024-11-22T03:41:16,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/.tmp/B/a1fded9ab59e445fa0393c9708538935 as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/B/a1fded9ab59e445fa0393c9708538935 2024-11-22T03:41:16,599 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/B/a1fded9ab59e445fa0393c9708538935, entries=150, sequenceid=351, filesize=12.0 K 2024-11-22T03:41:16,600 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/.tmp/C/2d79847d3a1e48af991987147964ed29 as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/C/2d79847d3a1e48af991987147964ed29 2024-11-22T03:41:16,605 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/C/2d79847d3a1e48af991987147964ed29, entries=150, sequenceid=351, filesize=12.0 K 2024-11-22T03:41:16,605 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~123.75 KB/126720, currentSize=154.31 KB/158010 for f0af25cc84372b1efb01f18522e2d562 in 1269ms, sequenceid=351, compaction requested=true 2024-11-22T03:41:16,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2603): Flush status journal for f0af25cc84372b1efb01f18522e2d562: 2024-11-22T03:41:16,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562. 2024-11-22T03:41:16,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=79 2024-11-22T03:41:16,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42627 {}] master.HMaster(4169): Remote procedure done, pid=79 2024-11-22T03:41:16,608 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=79, resume processing ppid=78 2024-11-22T03:41:16,608 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=79, ppid=78, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5760 sec 2024-11-22T03:41:16,610 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=78, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees in 1.5800 sec 2024-11-22T03:41:16,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(8855): Flush requested on f0af25cc84372b1efb01f18522e2d562 2024-11-22T03:41:16,770 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-22T03:41:16,770 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f0af25cc84372b1efb01f18522e2d562 3/3 column families, dataSize=161.02 KB heapSize=422.63 KB 2024-11-22T03:41:16,770 INFO [Thread-985 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-985. 2024-11-22T03:41:16,770 DEBUG [Thread-985 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-22T03:41:16,770 DEBUG [Thread-985 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T03:41:16,770 DEBUG [Thread-985 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T03:41:16,770 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-22T03:41:16,775 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/.tmp/A/29990e8a63bb44a095eb825b4a866eaa is 50, key is test_row_0/A:col10/1732246875628/Put/seqid=0 2024-11-22T03:41:16,778 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742106_1282 (size=12301) 2024-11-22T03:41:17,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-22T03:41:17,180 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-22T03:41:17,180 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=378 (bloomFilter=true), to=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/.tmp/A/29990e8a63bb44a095eb825b4a866eaa 2024-11-22T03:41:17,193 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/.tmp/B/cc297c79067c4b08b440565f17ec457c is 50, key is test_row_0/B:col10/1732246875628/Put/seqid=0 2024-11-22T03:41:17,196 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742107_1283 (size=12301) 2024-11-22T03:41:17,598 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=378 (bloomFilter=true), to=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/.tmp/B/cc297c79067c4b08b440565f17ec457c 2024-11-22T03:41:17,601 INFO [Thread-993 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-993. 2024-11-22T03:41:17,601 DEBUG [Thread-993 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-22T03:41:17,601 DEBUG [Thread-993 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T03:41:17,602 DEBUG [Thread-993 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T03:41:17,602 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-22T03:41:17,610 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/.tmp/C/eae24fb3dd3e415b8a200c0395b4b60e is 50, key is test_row_0/C:col10/1732246875628/Put/seqid=0 2024-11-22T03:41:17,614 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742108_1284 (size=12301) 2024-11-22T03:41:17,620 INFO [Thread-991 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-991. 2024-11-22T03:41:17,621 DEBUG [Thread-991 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-22T03:41:17,621 DEBUG [Thread-991 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T03:41:17,621 DEBUG [Thread-991 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T03:41:17,621 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-22T03:41:18,016 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=378 (bloomFilter=true), to=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/.tmp/C/eae24fb3dd3e415b8a200c0395b4b60e 2024-11-22T03:41:18,025 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/.tmp/A/29990e8a63bb44a095eb825b4a866eaa as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/A/29990e8a63bb44a095eb825b4a866eaa 2024-11-22T03:41:18,029 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/A/29990e8a63bb44a095eb825b4a866eaa, entries=150, sequenceid=378, filesize=12.0 K 2024-11-22T03:41:18,029 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/.tmp/B/cc297c79067c4b08b440565f17ec457c as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/B/cc297c79067c4b08b440565f17ec457c 2024-11-22T03:41:18,032 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/B/cc297c79067c4b08b440565f17ec457c, entries=150, sequenceid=378, filesize=12.0 K 2024-11-22T03:41:18,033 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/.tmp/C/eae24fb3dd3e415b8a200c0395b4b60e as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/C/eae24fb3dd3e415b8a200c0395b4b60e 2024-11-22T03:41:18,036 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/C/eae24fb3dd3e415b8a200c0395b4b60e, entries=150, sequenceid=378, filesize=12.0 K 2024-11-22T03:41:18,037 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~422.58 KB/432720, currentSize=13.42 KB/13740 for f0af25cc84372b1efb01f18522e2d562 in 1267ms, sequenceid=378, compaction requested=true 2024-11-22T03:41:18,037 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f0af25cc84372b1efb01f18522e2d562: 2024-11-22T03:41:18,037 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f0af25cc84372b1efb01f18522e2d562:A, priority=-2147483648, current under compaction store size is 1 2024-11-22T03:41:18,037 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T03:41:18,037 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f0af25cc84372b1efb01f18522e2d562:B, priority=-2147483648, current under compaction store size is 2 2024-11-22T03:41:18,037 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T03:41:18,037 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-22T03:41:18,037 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f0af25cc84372b1efb01f18522e2d562:C, priority=-2147483648, current under compaction store size is 3 2024-11-22T03:41:18,037 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-22T03:41:18,037 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T03:41:18,038 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49954 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-22T03:41:18,038 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49954 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-22T03:41:18,038 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HStore(1541): f0af25cc84372b1efb01f18522e2d562/B is initiating minor compaction (all files) 2024-11-22T03:41:18,038 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.HStore(1541): f0af25cc84372b1efb01f18522e2d562/A is initiating minor compaction (all files) 2024-11-22T03:41:18,038 INFO [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f0af25cc84372b1efb01f18522e2d562/B in TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562. 2024-11-22T03:41:18,038 INFO [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f0af25cc84372b1efb01f18522e2d562/A in TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562. 2024-11-22T03:41:18,039 INFO [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/B/87390e5b4d20420b81efe8d95e7447ff, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/B/9c10da7702884f8c9ab2f501d6e370e6, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/B/a1fded9ab59e445fa0393c9708538935, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/B/cc297c79067c4b08b440565f17ec457c] into tmpdir=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/.tmp, totalSize=48.8 K 2024-11-22T03:41:18,039 INFO [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/A/74716376f38144ea96f895cc1c170fab, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/A/8c92da645d5946559eff862a0b4ca3ca, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/A/fb33ac7c77f640a397daa70d4d471fc4, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/A/29990e8a63bb44a095eb825b4a866eaa] into tmpdir=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/.tmp, totalSize=48.8 K 2024-11-22T03:41:18,039 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] compactions.Compactor(225): Compacting 87390e5b4d20420b81efe8d95e7447ff, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=312, earliestPutTs=1732246873128 2024-11-22T03:41:18,039 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] compactions.Compactor(225): Compacting 74716376f38144ea96f895cc1c170fab, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=312, earliestPutTs=1732246873128 2024-11-22T03:41:18,039 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8c92da645d5946559eff862a0b4ca3ca, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=340, earliestPutTs=1732246873217 2024-11-22T03:41:18,039 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] compactions.Compactor(225): Compacting 9c10da7702884f8c9ab2f501d6e370e6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=340, earliestPutTs=1732246873217 2024-11-22T03:41:18,039 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] compactions.Compactor(225): Compacting fb33ac7c77f640a397daa70d4d471fc4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=351, earliestPutTs=1732246874387 2024-11-22T03:41:18,039 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] compactions.Compactor(225): Compacting a1fded9ab59e445fa0393c9708538935, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=351, earliestPutTs=1732246874387 2024-11-22T03:41:18,040 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] compactions.Compactor(225): Compacting 29990e8a63bb44a095eb825b4a866eaa, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=378, earliestPutTs=1732246875608 2024-11-22T03:41:18,040 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] compactions.Compactor(225): Compacting cc297c79067c4b08b440565f17ec457c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=378, earliestPutTs=1732246875608 2024-11-22T03:41:18,049 INFO [RS:0;c85114ed5096:42545-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f0af25cc84372b1efb01f18522e2d562#A#compaction#250 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-22T03:41:18,049 INFO [RS:0;c85114ed5096:42545-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f0af25cc84372b1efb01f18522e2d562#B#compaction#249 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-22T03:41:18,049 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/.tmp/A/360a4baa9d9a4104b0b1c3a913496dca is 50, key is test_row_0/A:col10/1732246875628/Put/seqid=0 2024-11-22T03:41:18,049 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/.tmp/B/3f0875aa36ad4c49a6981415ba1d7b9f is 50, key is test_row_0/B:col10/1732246875628/Put/seqid=0 2024-11-22T03:41:18,053 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742109_1285 (size=13187) 2024-11-22T03:41:18,053 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742110_1286 (size=13187) 2024-11-22T03:41:18,252 INFO [Thread-989 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-989. 2024-11-22T03:41:18,253 DEBUG [Thread-989 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-22T03:41:18,253 DEBUG [Thread-989 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T03:41:18,253 DEBUG [Thread-989 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T03:41:18,253 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-22T03:41:18,312 INFO [Thread-987 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-987. 2024-11-22T03:41:18,312 DEBUG [Thread-987 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-22T03:41:18,312 DEBUG [Thread-987 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T03:41:18,313 DEBUG [Thread-987 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T03:41:18,313 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-22T03:41:18,313 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-22T03:41:18,313 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 173 2024-11-22T03:41:18,313 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 25 2024-11-22T03:41:18,313 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 22 2024-11-22T03:41:18,313 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 38 2024-11-22T03:41:18,314 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 44 2024-11-22T03:41:18,314 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-22T03:41:18,314 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4456 2024-11-22T03:41:18,314 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4708 2024-11-22T03:41:18,314 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4341 2024-11-22T03:41:18,314 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4438 2024-11-22T03:41:18,314 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4709 2024-11-22T03:41:18,314 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-22T03:41:18,314 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-22T03:41:18,314 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testGetAtomicity(AcidGuaranteesTestBase.java:107) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-22T03:41:18,314 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T03:41:18,315 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T03:41:18,315 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-22T03:41:18,316 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42627 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-22T03:41:18,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42627 {}] procedure2.ProcedureExecutor(1139): Stored pid=80, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-22T03:41:18,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-22T03:41:18,322 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732246878322"}]},"ts":"1732246878322"} 2024-11-22T03:41:18,324 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-22T03:41:18,324 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-22T03:41:18,325 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=81, ppid=80, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-22T03:41:18,327 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=82, ppid=81, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f0af25cc84372b1efb01f18522e2d562, UNASSIGN}] 2024-11-22T03:41:18,328 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=82, ppid=81, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f0af25cc84372b1efb01f18522e2d562, UNASSIGN 2024-11-22T03:41:18,329 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=82 updating hbase:meta row=f0af25cc84372b1efb01f18522e2d562, regionState=CLOSING, regionLocation=c85114ed5096,42545,1732246786482 2024-11-22T03:41:18,331 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=82, ppid=81, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=f0af25cc84372b1efb01f18522e2d562, UNASSIGN because future has completed 2024-11-22T03:41:18,331 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-22T03:41:18,331 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=83, ppid=82, state=RUNNABLE, hasLock=false; CloseRegionProcedure f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482}] 2024-11-22T03:41:18,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-22T03:41:18,461 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/.tmp/A/360a4baa9d9a4104b0b1c3a913496dca as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/A/360a4baa9d9a4104b0b1c3a913496dca 2024-11-22T03:41:18,461 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/.tmp/B/3f0875aa36ad4c49a6981415ba1d7b9f as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/B/3f0875aa36ad4c49a6981415ba1d7b9f 2024-11-22T03:41:18,468 INFO [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in f0af25cc84372b1efb01f18522e2d562/B of f0af25cc84372b1efb01f18522e2d562 into 3f0875aa36ad4c49a6981415ba1d7b9f(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T03:41:18,468 INFO [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in f0af25cc84372b1efb01f18522e2d562/A of f0af25cc84372b1efb01f18522e2d562 into 360a4baa9d9a4104b0b1c3a913496dca(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T03:41:18,468 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f0af25cc84372b1efb01f18522e2d562: 2024-11-22T03:41:18,468 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f0af25cc84372b1efb01f18522e2d562: 2024-11-22T03:41:18,469 INFO [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., storeName=f0af25cc84372b1efb01f18522e2d562/B, priority=12, startTime=1732246878037; duration=0sec 2024-11-22T03:41:18,469 INFO [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., storeName=f0af25cc84372b1efb01f18522e2d562/A, priority=12, startTime=1732246878037; duration=0sec 2024-11-22T03:41:18,469 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T03:41:18,469 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f0af25cc84372b1efb01f18522e2d562:B 2024-11-22T03:41:18,469 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T03:41:18,469 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-22T03:41:18,469 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f0af25cc84372b1efb01f18522e2d562:A 2024-11-22T03:41:18,470 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49954 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-22T03:41:18,470 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HStore(1541): f0af25cc84372b1efb01f18522e2d562/C is initiating minor compaction (all files) 2024-11-22T03:41:18,471 INFO [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f0af25cc84372b1efb01f18522e2d562/C in TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562. 2024-11-22T03:41:18,471 INFO [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/C/9c9c7a7d54794d66872a2d1dfa9d159e, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/C/1746a321d3c2404bba9594cb37cdf442, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/C/2d79847d3a1e48af991987147964ed29, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/C/eae24fb3dd3e415b8a200c0395b4b60e] into tmpdir=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/.tmp, totalSize=48.8 K 2024-11-22T03:41:18,471 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] compactions.Compactor(225): Compacting 9c9c7a7d54794d66872a2d1dfa9d159e, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=312, earliestPutTs=1732246873128 2024-11-22T03:41:18,471 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] compactions.Compactor(225): Compacting 1746a321d3c2404bba9594cb37cdf442, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=340, earliestPutTs=1732246873217 2024-11-22T03:41:18,472 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] compactions.Compactor(225): Compacting 2d79847d3a1e48af991987147964ed29, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=351, earliestPutTs=1732246874387 2024-11-22T03:41:18,472 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] compactions.Compactor(225): Compacting eae24fb3dd3e415b8a200c0395b4b60e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=378, earliestPutTs=1732246875608 2024-11-22T03:41:18,482 INFO [RS:0;c85114ed5096:42545-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f0af25cc84372b1efb01f18522e2d562#C#compaction#251 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-22T03:41:18,483 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/.tmp/C/c2c47c06ac444b54a49c9e978270de90 is 50, key is test_row_0/C:col10/1732246875628/Put/seqid=0 2024-11-22T03:41:18,484 INFO [RS_CLOSE_REGION-regionserver/c85114ed5096:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] handler.UnassignRegionHandler(122): Close f0af25cc84372b1efb01f18522e2d562 2024-11-22T03:41:18,484 DEBUG [RS_CLOSE_REGION-regionserver/c85114ed5096:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-22T03:41:18,484 DEBUG [RS_CLOSE_REGION-regionserver/c85114ed5096:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegion(1722): Closing f0af25cc84372b1efb01f18522e2d562, disabling compactions & flushes 2024-11-22T03:41:18,484 DEBUG [RS_CLOSE_REGION-regionserver/c85114ed5096:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegion(1993): waiting for 1 compactions to complete for region TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562. 2024-11-22T03:41:18,487 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742111_1287 (size=13187) 2024-11-22T03:41:18,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-22T03:41:18,892 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/.tmp/C/c2c47c06ac444b54a49c9e978270de90 as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/C/c2c47c06ac444b54a49c9e978270de90 2024-11-22T03:41:18,897 INFO [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in f0af25cc84372b1efb01f18522e2d562/C of f0af25cc84372b1efb01f18522e2d562 into c2c47c06ac444b54a49c9e978270de90(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T03:41:18,897 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f0af25cc84372b1efb01f18522e2d562: 2024-11-22T03:41:18,897 INFO [RS_CLOSE_REGION-regionserver/c85114ed5096:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562. 2024-11-22T03:41:18,897 INFO [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., storeName=f0af25cc84372b1efb01f18522e2d562/C, priority=12, startTime=1732246878037; duration=0sec 2024-11-22T03:41:18,897 DEBUG [RS_CLOSE_REGION-regionserver/c85114ed5096:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562. 2024-11-22T03:41:18,897 DEBUG [RS_CLOSE_REGION-regionserver/c85114ed5096:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562. after waiting 0 ms 2024-11-22T03:41:18,897 DEBUG [RS_CLOSE_REGION-regionserver/c85114ed5096:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562. 2024-11-22T03:41:18,897 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T03:41:18,897 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f0af25cc84372b1efb01f18522e2d562:C 2024-11-22T03:41:18,897 INFO [RS_CLOSE_REGION-regionserver/c85114ed5096:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegion(2902): Flushing f0af25cc84372b1efb01f18522e2d562 3/3 column families, dataSize=26.84 KB heapSize=71.06 KB 2024-11-22T03:41:18,901 DEBUG [RS_CLOSE_REGION-regionserver/c85114ed5096:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/.tmp/A/b0c0022f4a7b422ba1698837d910bbcc is 50, key is test_row_0/A:col10/1732246878250/Put/seqid=0 2024-11-22T03:41:18,905 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742112_1288 (size=12301) 2024-11-22T03:41:18,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-22T03:41:19,306 INFO [RS_CLOSE_REGION-regionserver/c85114ed5096:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=388 (bloomFilter=true), to=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/.tmp/A/b0c0022f4a7b422ba1698837d910bbcc 2024-11-22T03:41:19,317 DEBUG [RS_CLOSE_REGION-regionserver/c85114ed5096:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/.tmp/B/03155db72f9140cb990c1f4c2cb26979 is 50, key is test_row_0/B:col10/1732246878250/Put/seqid=0 2024-11-22T03:41:19,321 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742113_1289 (size=12301) 2024-11-22T03:41:19,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-22T03:41:19,722 INFO [RS_CLOSE_REGION-regionserver/c85114ed5096:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=388 (bloomFilter=true), to=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/.tmp/B/03155db72f9140cb990c1f4c2cb26979 2024-11-22T03:41:19,729 DEBUG [RS_CLOSE_REGION-regionserver/c85114ed5096:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/.tmp/C/099777028589426bb56eb7d4bf46c4c2 is 50, key is test_row_0/C:col10/1732246878250/Put/seqid=0 2024-11-22T03:41:19,732 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742114_1290 (size=12301) 2024-11-22T03:41:20,133 INFO [RS_CLOSE_REGION-regionserver/c85114ed5096:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=388 (bloomFilter=true), to=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/.tmp/C/099777028589426bb56eb7d4bf46c4c2 2024-11-22T03:41:20,142 DEBUG [RS_CLOSE_REGION-regionserver/c85114ed5096:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/.tmp/A/b0c0022f4a7b422ba1698837d910bbcc as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/A/b0c0022f4a7b422ba1698837d910bbcc 2024-11-22T03:41:20,148 INFO [RS_CLOSE_REGION-regionserver/c85114ed5096:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/A/b0c0022f4a7b422ba1698837d910bbcc, entries=150, sequenceid=388, filesize=12.0 K 2024-11-22T03:41:20,150 DEBUG [RS_CLOSE_REGION-regionserver/c85114ed5096:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/.tmp/B/03155db72f9140cb990c1f4c2cb26979 as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/B/03155db72f9140cb990c1f4c2cb26979 2024-11-22T03:41:20,155 INFO [RS_CLOSE_REGION-regionserver/c85114ed5096:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/B/03155db72f9140cb990c1f4c2cb26979, entries=150, sequenceid=388, filesize=12.0 K 2024-11-22T03:41:20,157 DEBUG [RS_CLOSE_REGION-regionserver/c85114ed5096:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/.tmp/C/099777028589426bb56eb7d4bf46c4c2 as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/C/099777028589426bb56eb7d4bf46c4c2 2024-11-22T03:41:20,162 INFO [RS_CLOSE_REGION-regionserver/c85114ed5096:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/C/099777028589426bb56eb7d4bf46c4c2, entries=150, sequenceid=388, filesize=12.0 K 2024-11-22T03:41:20,163 INFO [RS_CLOSE_REGION-regionserver/c85114ed5096:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~71.02 KB/72720, currentSize=0 B/0 for f0af25cc84372b1efb01f18522e2d562 in 1266ms, sequenceid=388, compaction requested=false 2024-11-22T03:41:20,164 DEBUG [StoreCloser-TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/A/8332a7a1776e4cc894739dace3dcea08, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/A/360dac24cf294908be14f8507c0a9cef, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/A/ce9c130885994d718d8cded9e821076d, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/A/5895b4596238410481d41ef161ba469a, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/A/0313ed4a34164ca48471ecb993831253, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/A/8553f5e1fd924c6cb0b35ce63adb703f, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/A/544b39b880cc4720b45461244ceeebb1, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/A/8795c14b3477408a86ea2056dd8149a5, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/A/4c0b7a3807214367bf4d9d9b94354ab7, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/A/7a4710f466d5403a84c7cedf0f23df80, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/A/576e023f019d4aaeaddfcdcc40ef8a24, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/A/6395378272cc4904a0e8928dfec2c720, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/A/c0e83b1eb5334578a7e6871eae8a2565, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/A/e7dfeff5273b4f4f83624040bed8557d, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/A/21c14fc0778a4e34988a98ff861410ad, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/A/d2bdfe5f6b534c8c9482bf20d51afc16, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/A/d040c025d62e44988c9596e31e8b49e2, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/A/4e43a5bc1c7e44fdbe5219c2fae74724, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/A/fb9c8e9a99834bdaa04449c7fe661802, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/A/940e4b592a99498292b706626c494285, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/A/2f88255cd7a34a67a114fcb3a3b99e88, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/A/0744fbb81b434e749b597f834b1038bc, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/A/74716376f38144ea96f895cc1c170fab, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/A/8c92da645d5946559eff862a0b4ca3ca, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/A/fb33ac7c77f640a397daa70d4d471fc4, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/A/29990e8a63bb44a095eb825b4a866eaa] to archive 2024-11-22T03:41:20,165 DEBUG [StoreCloser-TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-22T03:41:20,168 DEBUG [StoreCloser-TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/A/8332a7a1776e4cc894739dace3dcea08 to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/A/8332a7a1776e4cc894739dace3dcea08 2024-11-22T03:41:20,169 DEBUG [StoreCloser-TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/A/360dac24cf294908be14f8507c0a9cef to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/A/360dac24cf294908be14f8507c0a9cef 2024-11-22T03:41:20,170 DEBUG [StoreCloser-TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/A/ce9c130885994d718d8cded9e821076d to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/A/ce9c130885994d718d8cded9e821076d 2024-11-22T03:41:20,171 DEBUG [StoreCloser-TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/A/5895b4596238410481d41ef161ba469a to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/A/5895b4596238410481d41ef161ba469a 2024-11-22T03:41:20,172 DEBUG [StoreCloser-TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/A/0313ed4a34164ca48471ecb993831253 to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/A/0313ed4a34164ca48471ecb993831253 2024-11-22T03:41:20,173 DEBUG [StoreCloser-TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/A/8553f5e1fd924c6cb0b35ce63adb703f to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/A/8553f5e1fd924c6cb0b35ce63adb703f 2024-11-22T03:41:20,174 DEBUG [StoreCloser-TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/A/544b39b880cc4720b45461244ceeebb1 to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/A/544b39b880cc4720b45461244ceeebb1 2024-11-22T03:41:20,175 DEBUG [StoreCloser-TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/A/8795c14b3477408a86ea2056dd8149a5 to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/A/8795c14b3477408a86ea2056dd8149a5 2024-11-22T03:41:20,176 DEBUG [StoreCloser-TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/A/4c0b7a3807214367bf4d9d9b94354ab7 to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/A/4c0b7a3807214367bf4d9d9b94354ab7 2024-11-22T03:41:20,177 DEBUG [StoreCloser-TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/A/7a4710f466d5403a84c7cedf0f23df80 to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/A/7a4710f466d5403a84c7cedf0f23df80 2024-11-22T03:41:20,178 DEBUG [StoreCloser-TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/A/576e023f019d4aaeaddfcdcc40ef8a24 to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/A/576e023f019d4aaeaddfcdcc40ef8a24 2024-11-22T03:41:20,178 DEBUG [StoreCloser-TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/A/6395378272cc4904a0e8928dfec2c720 to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/A/6395378272cc4904a0e8928dfec2c720 2024-11-22T03:41:20,179 DEBUG [StoreCloser-TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/A/c0e83b1eb5334578a7e6871eae8a2565 to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/A/c0e83b1eb5334578a7e6871eae8a2565 2024-11-22T03:41:20,180 DEBUG [StoreCloser-TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/A/e7dfeff5273b4f4f83624040bed8557d to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/A/e7dfeff5273b4f4f83624040bed8557d 2024-11-22T03:41:20,180 DEBUG [StoreCloser-TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/A/21c14fc0778a4e34988a98ff861410ad to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/A/21c14fc0778a4e34988a98ff861410ad 2024-11-22T03:41:20,181 DEBUG [StoreCloser-TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/A/d2bdfe5f6b534c8c9482bf20d51afc16 to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/A/d2bdfe5f6b534c8c9482bf20d51afc16 2024-11-22T03:41:20,182 DEBUG [StoreCloser-TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/A/d040c025d62e44988c9596e31e8b49e2 to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/A/d040c025d62e44988c9596e31e8b49e2 2024-11-22T03:41:20,183 DEBUG [StoreCloser-TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/A/4e43a5bc1c7e44fdbe5219c2fae74724 to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/A/4e43a5bc1c7e44fdbe5219c2fae74724 2024-11-22T03:41:20,183 DEBUG [StoreCloser-TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/A/fb9c8e9a99834bdaa04449c7fe661802 to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/A/fb9c8e9a99834bdaa04449c7fe661802 2024-11-22T03:41:20,184 DEBUG [StoreCloser-TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/A/940e4b592a99498292b706626c494285 to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/A/940e4b592a99498292b706626c494285 2024-11-22T03:41:20,185 DEBUG [StoreCloser-TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/A/2f88255cd7a34a67a114fcb3a3b99e88 to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/A/2f88255cd7a34a67a114fcb3a3b99e88 2024-11-22T03:41:20,186 DEBUG [StoreCloser-TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/A/0744fbb81b434e749b597f834b1038bc to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/A/0744fbb81b434e749b597f834b1038bc 2024-11-22T03:41:20,187 DEBUG [StoreCloser-TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/A/74716376f38144ea96f895cc1c170fab to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/A/74716376f38144ea96f895cc1c170fab 2024-11-22T03:41:20,187 DEBUG [StoreCloser-TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/A/8c92da645d5946559eff862a0b4ca3ca to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/A/8c92da645d5946559eff862a0b4ca3ca 2024-11-22T03:41:20,188 DEBUG [StoreCloser-TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/A/fb33ac7c77f640a397daa70d4d471fc4 to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/A/fb33ac7c77f640a397daa70d4d471fc4 2024-11-22T03:41:20,189 DEBUG [StoreCloser-TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/A/29990e8a63bb44a095eb825b4a866eaa to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/A/29990e8a63bb44a095eb825b4a866eaa 2024-11-22T03:41:20,190 DEBUG [StoreCloser-TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/B/0602f6ff1008455e94d81b162d6d8c3b, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/B/8c6ce8a97d054672a56b1e4b6ccfeede, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/B/597b6c5cb45c4396a1ed47468cc82c9b, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/B/3eca758af85c41278c085e6a74467150, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/B/df06dbea9aad4262aa4376083616dfdf, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/B/fb8942f6a96d4d49999f5324477d3a6e, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/B/3e4065072da944e3afdec445fc6f29e8, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/B/da2c292ff1d049a98249aa78cbe4916b, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/B/5b09921f55c84d6fa1635753b0e06309, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/B/4f698dbb5cfe4aa69e747bfe37e997d1, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/B/691c750a61904a7fa9e41cb5324a1dba, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/B/13789e4827384b3bb7b113f1ddaea36e, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/B/96f0fc7b078a4512bc5f4705da404a2e, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/B/0ef60703a1e140c1a14e6a9717b4b42a, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/B/9d3c752c288140f99b9a2e93e576d3cd, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/B/a529d6e5713d48b1855ee23456d3a775, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/B/864d7ce5710243e0a9ab7eb1306c5507, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/B/5c63f0f2f42f4820b9670c6ef1f2c10f, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/B/e6d819e644a1413a87b34ea930ddb363, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/B/9f9f64fb1a364458ac050c2c2d400d9e, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/B/8ac49ea535504f1e88601386d3c01011, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/B/87390e5b4d20420b81efe8d95e7447ff, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/B/b2a4f05e734b435daef1ad0203c1e978, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/B/9c10da7702884f8c9ab2f501d6e370e6, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/B/a1fded9ab59e445fa0393c9708538935, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/B/cc297c79067c4b08b440565f17ec457c] to archive 2024-11-22T03:41:20,190 DEBUG [StoreCloser-TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-22T03:41:20,191 DEBUG [StoreCloser-TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/B/0602f6ff1008455e94d81b162d6d8c3b to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/B/0602f6ff1008455e94d81b162d6d8c3b 2024-11-22T03:41:20,192 DEBUG [StoreCloser-TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/B/8c6ce8a97d054672a56b1e4b6ccfeede to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/B/8c6ce8a97d054672a56b1e4b6ccfeede 2024-11-22T03:41:20,193 DEBUG [StoreCloser-TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/B/597b6c5cb45c4396a1ed47468cc82c9b to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/B/597b6c5cb45c4396a1ed47468cc82c9b 2024-11-22T03:41:20,193 DEBUG [StoreCloser-TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/B/3eca758af85c41278c085e6a74467150 to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/B/3eca758af85c41278c085e6a74467150 2024-11-22T03:41:20,194 DEBUG [StoreCloser-TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/B/df06dbea9aad4262aa4376083616dfdf to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/B/df06dbea9aad4262aa4376083616dfdf 2024-11-22T03:41:20,195 DEBUG [StoreCloser-TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/B/fb8942f6a96d4d49999f5324477d3a6e to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/B/fb8942f6a96d4d49999f5324477d3a6e 2024-11-22T03:41:20,195 DEBUG [StoreCloser-TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/B/3e4065072da944e3afdec445fc6f29e8 to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/B/3e4065072da944e3afdec445fc6f29e8 2024-11-22T03:41:20,196 DEBUG [StoreCloser-TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/B/da2c292ff1d049a98249aa78cbe4916b to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/B/da2c292ff1d049a98249aa78cbe4916b 2024-11-22T03:41:20,196 DEBUG [StoreCloser-TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/B/5b09921f55c84d6fa1635753b0e06309 to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/B/5b09921f55c84d6fa1635753b0e06309 2024-11-22T03:41:20,197 DEBUG [StoreCloser-TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/B/4f698dbb5cfe4aa69e747bfe37e997d1 to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/B/4f698dbb5cfe4aa69e747bfe37e997d1 2024-11-22T03:41:20,198 DEBUG [StoreCloser-TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/B/691c750a61904a7fa9e41cb5324a1dba to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/B/691c750a61904a7fa9e41cb5324a1dba 2024-11-22T03:41:20,199 DEBUG [StoreCloser-TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/B/13789e4827384b3bb7b113f1ddaea36e to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/B/13789e4827384b3bb7b113f1ddaea36e 2024-11-22T03:41:20,199 DEBUG [StoreCloser-TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/B/96f0fc7b078a4512bc5f4705da404a2e to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/B/96f0fc7b078a4512bc5f4705da404a2e 2024-11-22T03:41:20,200 DEBUG [StoreCloser-TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/B/0ef60703a1e140c1a14e6a9717b4b42a to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/B/0ef60703a1e140c1a14e6a9717b4b42a 2024-11-22T03:41:20,201 DEBUG [StoreCloser-TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/B/9d3c752c288140f99b9a2e93e576d3cd to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/B/9d3c752c288140f99b9a2e93e576d3cd 2024-11-22T03:41:20,201 DEBUG [StoreCloser-TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/B/a529d6e5713d48b1855ee23456d3a775 to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/B/a529d6e5713d48b1855ee23456d3a775 2024-11-22T03:41:20,202 DEBUG [StoreCloser-TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/B/864d7ce5710243e0a9ab7eb1306c5507 to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/B/864d7ce5710243e0a9ab7eb1306c5507 2024-11-22T03:41:20,203 DEBUG [StoreCloser-TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/B/5c63f0f2f42f4820b9670c6ef1f2c10f to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/B/5c63f0f2f42f4820b9670c6ef1f2c10f 2024-11-22T03:41:20,203 DEBUG [StoreCloser-TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/B/e6d819e644a1413a87b34ea930ddb363 to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/B/e6d819e644a1413a87b34ea930ddb363 2024-11-22T03:41:20,204 DEBUG [StoreCloser-TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/B/9f9f64fb1a364458ac050c2c2d400d9e to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/B/9f9f64fb1a364458ac050c2c2d400d9e 2024-11-22T03:41:20,205 DEBUG [StoreCloser-TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/B/8ac49ea535504f1e88601386d3c01011 to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/B/8ac49ea535504f1e88601386d3c01011 2024-11-22T03:41:20,206 DEBUG [StoreCloser-TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/B/87390e5b4d20420b81efe8d95e7447ff to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/B/87390e5b4d20420b81efe8d95e7447ff 2024-11-22T03:41:20,206 DEBUG [StoreCloser-TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/B/b2a4f05e734b435daef1ad0203c1e978 to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/B/b2a4f05e734b435daef1ad0203c1e978 2024-11-22T03:41:20,207 DEBUG [StoreCloser-TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/B/9c10da7702884f8c9ab2f501d6e370e6 to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/B/9c10da7702884f8c9ab2f501d6e370e6 2024-11-22T03:41:20,208 DEBUG [StoreCloser-TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/B/a1fded9ab59e445fa0393c9708538935 to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/B/a1fded9ab59e445fa0393c9708538935 2024-11-22T03:41:20,208 DEBUG [StoreCloser-TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/B/cc297c79067c4b08b440565f17ec457c to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/B/cc297c79067c4b08b440565f17ec457c 2024-11-22T03:41:20,210 DEBUG [StoreCloser-TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/C/d0b829e9f5db4a1bbee3ee88b56800f2, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/C/c56fbdd13ddf4b9c9ae351299d7311b8, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/C/5df9af5387404f3888e0cd48ee9e075d, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/C/444c524511c246a193f089a5386fbc20, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/C/1d0ab2cfdb32417f938379e8d8fbc81f, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/C/9a5b3dbcec1f45358f56f8b9a360a457, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/C/958da13a301a4354aa1612841c3ce7d4, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/C/0d44541354de4805a659eb9abbcf7a19, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/C/04f21fdb166b404380b6ef47531afe11, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/C/1b3ee846b0af419da2a74cc498c06d9f, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/C/7b619c6f759a4e05b8234045512bb775, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/C/d02a0d89b3d04b78b59b7da1676d7770, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/C/aaeef659ae27433f80d35920dadb86ce, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/C/aa5b0362060e4bb78740f2b77f4b413c, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/C/8cc3162719944edea3667433d5827b83, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/C/e94c61045368464bb576a5e022cf9ae0, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/C/6dd9e1915ad143bfa36d8a1d194e99e7, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/C/223f6c0b772f476ea8383f7767b76aec, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/C/a0ab28bd3c5049b780a0c0fb25b0c2c6, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/C/26dd7574bcc841bca4de79647f1fe14a, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/C/3b89a8cb841f450ba9c2e9fbb1a08f61, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/C/9c9c7a7d54794d66872a2d1dfa9d159e, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/C/16ea1d93a16e4a5a8e8f6e7eb191e311, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/C/1746a321d3c2404bba9594cb37cdf442, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/C/2d79847d3a1e48af991987147964ed29, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/C/eae24fb3dd3e415b8a200c0395b4b60e] to archive 2024-11-22T03:41:20,210 DEBUG [StoreCloser-TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-22T03:41:20,211 DEBUG [StoreCloser-TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/C/d0b829e9f5db4a1bbee3ee88b56800f2 to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/C/d0b829e9f5db4a1bbee3ee88b56800f2 2024-11-22T03:41:20,212 DEBUG [StoreCloser-TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/C/c56fbdd13ddf4b9c9ae351299d7311b8 to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/C/c56fbdd13ddf4b9c9ae351299d7311b8 2024-11-22T03:41:20,213 DEBUG [StoreCloser-TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/C/5df9af5387404f3888e0cd48ee9e075d to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/C/5df9af5387404f3888e0cd48ee9e075d 2024-11-22T03:41:20,214 DEBUG [StoreCloser-TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/C/444c524511c246a193f089a5386fbc20 to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/C/444c524511c246a193f089a5386fbc20 2024-11-22T03:41:20,215 DEBUG [StoreCloser-TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/C/1d0ab2cfdb32417f938379e8d8fbc81f to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/C/1d0ab2cfdb32417f938379e8d8fbc81f 2024-11-22T03:41:20,215 DEBUG [StoreCloser-TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/C/9a5b3dbcec1f45358f56f8b9a360a457 to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/C/9a5b3dbcec1f45358f56f8b9a360a457 2024-11-22T03:41:20,216 DEBUG [StoreCloser-TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/C/958da13a301a4354aa1612841c3ce7d4 to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/C/958da13a301a4354aa1612841c3ce7d4 2024-11-22T03:41:20,217 DEBUG [StoreCloser-TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/C/0d44541354de4805a659eb9abbcf7a19 to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/C/0d44541354de4805a659eb9abbcf7a19 2024-11-22T03:41:20,218 DEBUG [StoreCloser-TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/C/04f21fdb166b404380b6ef47531afe11 to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/C/04f21fdb166b404380b6ef47531afe11 2024-11-22T03:41:20,218 DEBUG [StoreCloser-TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/C/1b3ee846b0af419da2a74cc498c06d9f to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/C/1b3ee846b0af419da2a74cc498c06d9f 2024-11-22T03:41:20,219 DEBUG [StoreCloser-TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/C/7b619c6f759a4e05b8234045512bb775 to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/C/7b619c6f759a4e05b8234045512bb775 2024-11-22T03:41:20,220 DEBUG [StoreCloser-TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/C/d02a0d89b3d04b78b59b7da1676d7770 to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/C/d02a0d89b3d04b78b59b7da1676d7770 2024-11-22T03:41:20,220 DEBUG [StoreCloser-TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/C/aaeef659ae27433f80d35920dadb86ce to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/C/aaeef659ae27433f80d35920dadb86ce 2024-11-22T03:41:20,221 DEBUG [StoreCloser-TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/C/aa5b0362060e4bb78740f2b77f4b413c to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/C/aa5b0362060e4bb78740f2b77f4b413c 2024-11-22T03:41:20,222 DEBUG [StoreCloser-TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/C/8cc3162719944edea3667433d5827b83 to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/C/8cc3162719944edea3667433d5827b83 2024-11-22T03:41:20,223 DEBUG [StoreCloser-TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/C/e94c61045368464bb576a5e022cf9ae0 to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/C/e94c61045368464bb576a5e022cf9ae0 2024-11-22T03:41:20,223 DEBUG [StoreCloser-TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/C/6dd9e1915ad143bfa36d8a1d194e99e7 to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/C/6dd9e1915ad143bfa36d8a1d194e99e7 2024-11-22T03:41:20,224 DEBUG [StoreCloser-TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/C/223f6c0b772f476ea8383f7767b76aec to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/C/223f6c0b772f476ea8383f7767b76aec 2024-11-22T03:41:20,225 DEBUG [StoreCloser-TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/C/a0ab28bd3c5049b780a0c0fb25b0c2c6 to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/C/a0ab28bd3c5049b780a0c0fb25b0c2c6 2024-11-22T03:41:20,226 DEBUG [StoreCloser-TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/C/26dd7574bcc841bca4de79647f1fe14a to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/C/26dd7574bcc841bca4de79647f1fe14a 2024-11-22T03:41:20,227 DEBUG [StoreCloser-TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/C/3b89a8cb841f450ba9c2e9fbb1a08f61 to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/C/3b89a8cb841f450ba9c2e9fbb1a08f61 2024-11-22T03:41:20,227 DEBUG [StoreCloser-TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/C/9c9c7a7d54794d66872a2d1dfa9d159e to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/C/9c9c7a7d54794d66872a2d1dfa9d159e 2024-11-22T03:41:20,228 DEBUG [StoreCloser-TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/C/16ea1d93a16e4a5a8e8f6e7eb191e311 to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/C/16ea1d93a16e4a5a8e8f6e7eb191e311 2024-11-22T03:41:20,229 DEBUG [StoreCloser-TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/C/1746a321d3c2404bba9594cb37cdf442 to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/C/1746a321d3c2404bba9594cb37cdf442 2024-11-22T03:41:20,230 DEBUG [StoreCloser-TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/C/2d79847d3a1e48af991987147964ed29 to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/C/2d79847d3a1e48af991987147964ed29 2024-11-22T03:41:20,230 DEBUG [StoreCloser-TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/C/eae24fb3dd3e415b8a200c0395b4b60e to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/C/eae24fb3dd3e415b8a200c0395b4b60e 2024-11-22T03:41:20,234 DEBUG [RS_CLOSE_REGION-regionserver/c85114ed5096:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/recovered.edits/391.seqid, newMaxSeqId=391, maxSeqId=1 2024-11-22T03:41:20,234 INFO [RS_CLOSE_REGION-regionserver/c85114ed5096:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562. 2024-11-22T03:41:20,235 DEBUG [RS_CLOSE_REGION-regionserver/c85114ed5096:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegion(1676): Region close journal for f0af25cc84372b1efb01f18522e2d562: Waiting for close lock at 1732246878484Running coprocessor pre-close hooks at 1732246878484Disabling compacts and flushes for region at 1732246878484Disabling writes for close at 1732246878897 (+413 ms)Obtaining lock to block concurrent updates at 1732246878897Preparing flush snapshotting stores in f0af25cc84372b1efb01f18522e2d562 at 1732246878897Finished memstore snapshotting TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562., syncing WAL and waiting on mvcc, flushsize=dataSize=27480, getHeapSize=72720, getOffHeapSize=0, getCellsCount=600 at 1732246878898 (+1 ms)Flushing stores of TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562. at 1732246878898Flushing f0af25cc84372b1efb01f18522e2d562/A: creating writer at 1732246878898Flushing f0af25cc84372b1efb01f18522e2d562/A: appending metadata at 1732246878901 (+3 ms)Flushing f0af25cc84372b1efb01f18522e2d562/A: closing flushed file at 1732246878901Flushing f0af25cc84372b1efb01f18522e2d562/B: creating writer at 1732246879314 (+413 ms)Flushing f0af25cc84372b1efb01f18522e2d562/B: appending metadata at 1732246879317 (+3 ms)Flushing f0af25cc84372b1efb01f18522e2d562/B: closing flushed file at 1732246879317Flushing f0af25cc84372b1efb01f18522e2d562/C: creating writer at 1732246879726 (+409 ms)Flushing f0af25cc84372b1efb01f18522e2d562/C: appending metadata at 1732246879728 (+2 ms)Flushing f0af25cc84372b1efb01f18522e2d562/C: closing flushed file at 1732246879728Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@37c383a3: reopening flushed file at 1732246880141 (+413 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@64c893e7: reopening flushed file at 1732246880148 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1d6b7ee: reopening flushed file at 1732246880156 (+8 ms)Finished flush of dataSize ~26.84 KB/27480, heapSize ~71.02 KB/72720, currentSize=0 B/0 for f0af25cc84372b1efb01f18522e2d562 in 1266ms, sequenceid=388, compaction requested=false at 1732246880163 (+7 ms)Writing region close event to WAL at 1732246880232 (+69 ms)Running coprocessor post-close hooks at 1732246880234 (+2 ms)Closed at 1732246880234 2024-11-22T03:41:20,236 INFO [RS_CLOSE_REGION-regionserver/c85114ed5096:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] handler.UnassignRegionHandler(157): Closed f0af25cc84372b1efb01f18522e2d562 2024-11-22T03:41:20,237 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=82 updating hbase:meta row=f0af25cc84372b1efb01f18522e2d562, regionState=CLOSED 2024-11-22T03:41:20,238 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=83, ppid=82, state=RUNNABLE, hasLock=false; CloseRegionProcedure f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 because future has completed 2024-11-22T03:41:20,239 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=83, resume processing ppid=82 2024-11-22T03:41:20,239 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=83, ppid=82, state=SUCCESS, hasLock=false; CloseRegionProcedure f0af25cc84372b1efb01f18522e2d562, server=c85114ed5096,42545,1732246786482 in 1.9070 sec 2024-11-22T03:41:20,240 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=82, resume processing ppid=81 2024-11-22T03:41:20,241 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=82, ppid=81, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f0af25cc84372b1efb01f18522e2d562, UNASSIGN in 1.9120 sec 2024-11-22T03:41:20,242 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=81, resume processing ppid=80 2024-11-22T03:41:20,242 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=81, ppid=80, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.9160 sec 2024-11-22T03:41:20,243 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732246880243"}]},"ts":"1732246880243"} 2024-11-22T03:41:20,244 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-22T03:41:20,244 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-22T03:41:20,245 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=80, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.9270 sec 2024-11-22T03:41:20,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-22T03:41:20,469 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-22T03:41:20,470 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42627 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-22T03:41:20,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42627 {}] procedure2.ProcedureExecutor(1139): Stored pid=84, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-22T03:41:20,472 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=84, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-22T03:41:20,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-22T03:41:20,473 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=84, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-22T03:41:20,476 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562 2024-11-22T03:41:20,479 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/A, FileablePath, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/B, FileablePath, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/C, FileablePath, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/recovered.edits] 2024-11-22T03:41:20,483 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/A/360a4baa9d9a4104b0b1c3a913496dca to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/A/360a4baa9d9a4104b0b1c3a913496dca 2024-11-22T03:41:20,485 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/A/b0c0022f4a7b422ba1698837d910bbcc to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/A/b0c0022f4a7b422ba1698837d910bbcc 2024-11-22T03:41:20,488 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/B/03155db72f9140cb990c1f4c2cb26979 to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/B/03155db72f9140cb990c1f4c2cb26979 2024-11-22T03:41:20,490 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/B/3f0875aa36ad4c49a6981415ba1d7b9f to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/B/3f0875aa36ad4c49a6981415ba1d7b9f 2024-11-22T03:41:20,491 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/C/099777028589426bb56eb7d4bf46c4c2 to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/C/099777028589426bb56eb7d4bf46c4c2 2024-11-22T03:41:20,492 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/C/c2c47c06ac444b54a49c9e978270de90 to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/C/c2c47c06ac444b54a49c9e978270de90 2024-11-22T03:41:20,494 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/recovered.edits/391.seqid to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562/recovered.edits/391.seqid 2024-11-22T03:41:20,495 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/f0af25cc84372b1efb01f18522e2d562 2024-11-22T03:41:20,495 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-22T03:41:20,497 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=84, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-22T03:41:20,499 WARN [PEWorker-4 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-22T03:41:20,501 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-22T03:41:20,502 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=84, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-22T03:41:20,502 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-22T03:41:20,502 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732246880502"}]},"ts":"9223372036854775807"} 2024-11-22T03:41:20,504 INFO [PEWorker-4 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-22T03:41:20,504 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => f0af25cc84372b1efb01f18522e2d562, NAME => 'TestAcidGuarantees,,1732246853429.f0af25cc84372b1efb01f18522e2d562.', STARTKEY => '', ENDKEY => ''}] 2024-11-22T03:41:20,504 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-22T03:41:20,504 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732246880504"}]},"ts":"9223372036854775807"} 2024-11-22T03:41:20,506 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-22T03:41:20,507 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(133): Finished pid=84, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-22T03:41:20,508 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=84, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 37 msec 2024-11-22T03:41:20,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-22T03:41:20,579 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-22T03:41:20,579 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-22T03:41:20,589 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testGetAtomicity Thread=242 (was 238) Potentially hanging thread: HFileArchiver-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1145069759_22 at /127.0.0.1:33198 [Waiting for operation #301] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1145069759_22 at /127.0.0.1:38608 [Waiting for operation #9] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1251162740_22 at /127.0.0.1:50488 [Waiting for operation #197] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1251162740_22 at /127.0.0.1:38620 [Waiting for operation #9] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=459 (was 449) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=358 (was 378), ProcessCount=11 (was 11), AvailableMemoryMB=2992 (was 3192) 2024-11-22T03:41:20,599 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testMobScanAtomicity Thread=242, OpenFileDescriptor=459, MaxFileDescriptor=1048576, SystemLoadAverage=358, ProcessCount=11, AvailableMemoryMB=2992 2024-11-22T03:41:20,600 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42627 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-22T03:41:20,601 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42627 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-22T03:41:20,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42627 {}] procedure2.ProcedureExecutor(1139): Stored pid=85, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-22T03:41:20,603 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=85, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-22T03:41:20,603 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:20,603 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42627 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 85 2024-11-22T03:41:20,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=85 2024-11-22T03:41:20,604 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=85, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-22T03:41:20,610 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742115_1291 (size=959) 2024-11-22T03:41:20,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=85 2024-11-22T03:41:20,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=85 2024-11-22T03:41:21,016 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 47b13a1c78111d05c87ac4dbad84cbd4, NAME => 'TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a 2024-11-22T03:41:21,026 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742116_1292 (size=53) 2024-11-22T03:41:21,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=85 2024-11-22T03:41:21,429 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-22T03:41:21,429 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 47b13a1c78111d05c87ac4dbad84cbd4, disabling compactions & flushes 2024-11-22T03:41:21,429 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4. 2024-11-22T03:41:21,429 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4. 2024-11-22T03:41:21,429 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4. after waiting 0 ms 2024-11-22T03:41:21,429 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4. 2024-11-22T03:41:21,429 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4. 2024-11-22T03:41:21,429 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 47b13a1c78111d05c87ac4dbad84cbd4: Waiting for close lock at 1732246881429Disabling compacts and flushes for region at 1732246881429Disabling writes for close at 1732246881429Writing region close event to WAL at 1732246881429Closed at 1732246881429 2024-11-22T03:41:21,431 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=85, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-22T03:41:21,432 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732246881431"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732246881431"}]},"ts":"1732246881431"} 2024-11-22T03:41:21,433 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-22T03:41:21,434 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=85, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-22T03:41:21,434 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732246881434"}]},"ts":"1732246881434"} 2024-11-22T03:41:21,435 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-22T03:41:21,436 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=86, ppid=85, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=47b13a1c78111d05c87ac4dbad84cbd4, ASSIGN}] 2024-11-22T03:41:21,436 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=86, ppid=85, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=47b13a1c78111d05c87ac4dbad84cbd4, ASSIGN 2024-11-22T03:41:21,437 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=86, ppid=85, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=47b13a1c78111d05c87ac4dbad84cbd4, ASSIGN; state=OFFLINE, location=c85114ed5096,42545,1732246786482; forceNewPlan=false, retain=false 2024-11-22T03:41:21,588 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=86 updating hbase:meta row=47b13a1c78111d05c87ac4dbad84cbd4, regionState=OPENING, regionLocation=c85114ed5096,42545,1732246786482 2024-11-22T03:41:21,589 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=86, ppid=85, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=47b13a1c78111d05c87ac4dbad84cbd4, ASSIGN because future has completed 2024-11-22T03:41:21,590 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=87, ppid=86, state=RUNNABLE, hasLock=false; OpenRegionProcedure 47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482}] 2024-11-22T03:41:21,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=85 2024-11-22T03:41:21,747 INFO [RS_OPEN_REGION-regionserver/c85114ed5096:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4. 2024-11-22T03:41:21,747 DEBUG [RS_OPEN_REGION-regionserver/c85114ed5096:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(7752): Opening region: {ENCODED => 47b13a1c78111d05c87ac4dbad84cbd4, NAME => 'TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4.', STARTKEY => '', ENDKEY => ''} 2024-11-22T03:41:21,748 DEBUG [RS_OPEN_REGION-regionserver/c85114ed5096:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 47b13a1c78111d05c87ac4dbad84cbd4 2024-11-22T03:41:21,748 DEBUG [RS_OPEN_REGION-regionserver/c85114ed5096:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-22T03:41:21,748 DEBUG [RS_OPEN_REGION-regionserver/c85114ed5096:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(7794): checking encryption for 47b13a1c78111d05c87ac4dbad84cbd4 2024-11-22T03:41:21,748 DEBUG [RS_OPEN_REGION-regionserver/c85114ed5096:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(7797): checking classloading for 47b13a1c78111d05c87ac4dbad84cbd4 2024-11-22T03:41:21,750 INFO [StoreOpener-47b13a1c78111d05c87ac4dbad84cbd4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 47b13a1c78111d05c87ac4dbad84cbd4 2024-11-22T03:41:21,753 INFO [StoreOpener-47b13a1c78111d05c87ac4dbad84cbd4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 47b13a1c78111d05c87ac4dbad84cbd4 columnFamilyName A 2024-11-22T03:41:21,753 DEBUG [StoreOpener-47b13a1c78111d05c87ac4dbad84cbd4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:21,754 INFO [StoreOpener-47b13a1c78111d05c87ac4dbad84cbd4-1 {}] regionserver.HStore(327): Store=47b13a1c78111d05c87ac4dbad84cbd4/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-22T03:41:21,754 INFO [StoreOpener-47b13a1c78111d05c87ac4dbad84cbd4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 47b13a1c78111d05c87ac4dbad84cbd4 2024-11-22T03:41:21,756 INFO [StoreOpener-47b13a1c78111d05c87ac4dbad84cbd4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 47b13a1c78111d05c87ac4dbad84cbd4 columnFamilyName B 2024-11-22T03:41:21,756 DEBUG [StoreOpener-47b13a1c78111d05c87ac4dbad84cbd4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:21,757 INFO [StoreOpener-47b13a1c78111d05c87ac4dbad84cbd4-1 {}] regionserver.HStore(327): Store=47b13a1c78111d05c87ac4dbad84cbd4/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-22T03:41:21,757 INFO [StoreOpener-47b13a1c78111d05c87ac4dbad84cbd4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 47b13a1c78111d05c87ac4dbad84cbd4 2024-11-22T03:41:21,760 INFO [StoreOpener-47b13a1c78111d05c87ac4dbad84cbd4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 47b13a1c78111d05c87ac4dbad84cbd4 columnFamilyName C 2024-11-22T03:41:21,760 DEBUG [StoreOpener-47b13a1c78111d05c87ac4dbad84cbd4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:21,761 INFO [StoreOpener-47b13a1c78111d05c87ac4dbad84cbd4-1 {}] regionserver.HStore(327): Store=47b13a1c78111d05c87ac4dbad84cbd4/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-22T03:41:21,761 DEBUG [RS_OPEN_REGION-regionserver/c85114ed5096:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(1038): replaying wal for 47b13a1c78111d05c87ac4dbad84cbd4 2024-11-22T03:41:21,762 DEBUG [RS_OPEN_REGION-regionserver/c85114ed5096:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4 2024-11-22T03:41:21,763 DEBUG [RS_OPEN_REGION-regionserver/c85114ed5096:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4 2024-11-22T03:41:21,765 DEBUG [RS_OPEN_REGION-regionserver/c85114ed5096:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(1048): stopping wal replay for 47b13a1c78111d05c87ac4dbad84cbd4 2024-11-22T03:41:21,765 DEBUG [RS_OPEN_REGION-regionserver/c85114ed5096:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(1060): Cleaning up temporary data for 47b13a1c78111d05c87ac4dbad84cbd4 2024-11-22T03:41:21,766 DEBUG [RS_OPEN_REGION-regionserver/c85114ed5096:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-22T03:41:21,767 DEBUG [RS_OPEN_REGION-regionserver/c85114ed5096:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(1093): writing seq id for 47b13a1c78111d05c87ac4dbad84cbd4 2024-11-22T03:41:21,770 DEBUG [RS_OPEN_REGION-regionserver/c85114ed5096:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-22T03:41:21,770 INFO [RS_OPEN_REGION-regionserver/c85114ed5096:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(1114): Opened 47b13a1c78111d05c87ac4dbad84cbd4; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64531195, jitterRate=-0.038410261273384094}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-22T03:41:21,770 DEBUG [RS_OPEN_REGION-regionserver/c85114ed5096:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 47b13a1c78111d05c87ac4dbad84cbd4 2024-11-22T03:41:21,771 DEBUG [RS_OPEN_REGION-regionserver/c85114ed5096:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(1006): Region open journal for 47b13a1c78111d05c87ac4dbad84cbd4: Running coprocessor pre-open hook at 1732246881748Writing region info on filesystem at 1732246881748Initializing all the Stores at 1732246881749 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732246881750 (+1 ms)Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732246881750Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732246881750Cleaning up temporary data from old regions at 1732246881765 (+15 ms)Running coprocessor post-open hooks at 1732246881770 (+5 ms)Region opened successfully at 1732246881771 (+1 ms) 2024-11-22T03:41:21,772 INFO [RS_OPEN_REGION-regionserver/c85114ed5096:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., pid=87, masterSystemTime=1732246881742 2024-11-22T03:41:21,774 DEBUG [RS_OPEN_REGION-regionserver/c85114ed5096:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4. 2024-11-22T03:41:21,774 INFO [RS_OPEN_REGION-regionserver/c85114ed5096:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4. 2024-11-22T03:41:21,775 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=86 updating hbase:meta row=47b13a1c78111d05c87ac4dbad84cbd4, regionState=OPEN, openSeqNum=2, regionLocation=c85114ed5096,42545,1732246786482 2024-11-22T03:41:21,776 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=87, ppid=86, state=RUNNABLE, hasLock=false; OpenRegionProcedure 47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 because future has completed 2024-11-22T03:41:21,780 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=87, resume processing ppid=86 2024-11-22T03:41:21,780 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=87, ppid=86, state=SUCCESS, hasLock=false; OpenRegionProcedure 47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 in 188 msec 2024-11-22T03:41:21,782 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=86, resume processing ppid=85 2024-11-22T03:41:21,782 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=86, ppid=85, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=47b13a1c78111d05c87ac4dbad84cbd4, ASSIGN in 345 msec 2024-11-22T03:41:21,783 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=85, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-22T03:41:21,783 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732246881783"}]},"ts":"1732246881783"} 2024-11-22T03:41:21,785 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-22T03:41:21,786 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=85, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-22T03:41:21,788 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=85, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1850 sec 2024-11-22T03:41:22,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=85 2024-11-22T03:41:22,750 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-22T03:41:22,752 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@283b5885, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T03:41:22,752 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c85114ed5096,42627,-1 for getting cluster id 2024-11-22T03:41:22,753 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-22T03:41:22,755 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '67035b7f-c649-44a8-9db7-17adff49cedb' 2024-11-22T03:41:22,755 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-22T03:41:22,755 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "67035b7f-c649-44a8-9db7-17adff49cedb" 2024-11-22T03:41:22,756 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@566b94fb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T03:41:22,756 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c85114ed5096,42627,-1] 2024-11-22T03:41:22,756 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-22T03:41:22,757 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T03:41:22,758 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53504, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-22T03:41:22,758 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2d7e27a5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T03:41:22,759 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-22T03:41:22,760 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c85114ed5096,42545,1732246786482, seqNum=-1] 2024-11-22T03:41:22,760 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-22T03:41:22,761 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52408, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-22T03:41:22,762 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-22T03:41:22,763 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is c85114ed5096,42627,1732246785796 2024-11-22T03:41:22,763 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@648e381e 2024-11-22T03:41:22,763 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-22T03:41:22,764 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53520, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-22T03:41:22,766 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42627 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-22T03:41:22,766 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42627 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-22T03:41:22,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42627 {}] procedure2.ProcedureExecutor(1139): Stored pid=88, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-22T03:41:22,779 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742117_1293 (size=995) 2024-11-22T03:41:23,185 DEBUG [PEWorker-1 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.959 2024-11-22T03:41:23,185 INFO [PEWorker-1 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.995 2024-11-22T03:41:23,190 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=89, ppid=88, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-22T03:41:23,192 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=90, ppid=89, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=47b13a1c78111d05c87ac4dbad84cbd4, REOPEN/MOVE}] 2024-11-22T03:41:23,193 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=90, ppid=89, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=47b13a1c78111d05c87ac4dbad84cbd4, REOPEN/MOVE 2024-11-22T03:41:23,194 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=90 updating hbase:meta row=47b13a1c78111d05c87ac4dbad84cbd4, regionState=CLOSING, regionLocation=c85114ed5096,42545,1732246786482 2024-11-22T03:41:23,195 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=90, ppid=89, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=47b13a1c78111d05c87ac4dbad84cbd4, REOPEN/MOVE because future has completed 2024-11-22T03:41:23,196 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-22T03:41:23,196 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=91, ppid=90, state=RUNNABLE, hasLock=false; CloseRegionProcedure 47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482}] 2024-11-22T03:41:23,348 INFO [RS_CLOSE_REGION-regionserver/c85114ed5096:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] handler.UnassignRegionHandler(122): Close 47b13a1c78111d05c87ac4dbad84cbd4 2024-11-22T03:41:23,348 DEBUG [RS_CLOSE_REGION-regionserver/c85114ed5096:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-22T03:41:23,349 DEBUG [RS_CLOSE_REGION-regionserver/c85114ed5096:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(1722): Closing 47b13a1c78111d05c87ac4dbad84cbd4, disabling compactions & flushes 2024-11-22T03:41:23,349 INFO [RS_CLOSE_REGION-regionserver/c85114ed5096:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4. 2024-11-22T03:41:23,349 DEBUG [RS_CLOSE_REGION-regionserver/c85114ed5096:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4. 2024-11-22T03:41:23,349 DEBUG [RS_CLOSE_REGION-regionserver/c85114ed5096:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4. after waiting 0 ms 2024-11-22T03:41:23,349 DEBUG [RS_CLOSE_REGION-regionserver/c85114ed5096:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4. 2024-11-22T03:41:23,354 DEBUG [RS_CLOSE_REGION-regionserver/c85114ed5096:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-22T03:41:23,355 INFO [RS_CLOSE_REGION-regionserver/c85114ed5096:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4. 2024-11-22T03:41:23,355 DEBUG [RS_CLOSE_REGION-regionserver/c85114ed5096:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(1676): Region close journal for 47b13a1c78111d05c87ac4dbad84cbd4: Waiting for close lock at 1732246883348Running coprocessor pre-close hooks at 1732246883348Disabling compacts and flushes for region at 1732246883348Disabling writes for close at 1732246883349 (+1 ms)Writing region close event to WAL at 1732246883349Running coprocessor post-close hooks at 1732246883355 (+6 ms)Closed at 1732246883355 2024-11-22T03:41:23,355 WARN [RS_CLOSE_REGION-regionserver/c85114ed5096:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegionServer(3299): Not adding moved region record: 47b13a1c78111d05c87ac4dbad84cbd4 to self. 2024-11-22T03:41:23,357 INFO [RS_CLOSE_REGION-regionserver/c85114ed5096:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] handler.UnassignRegionHandler(157): Closed 47b13a1c78111d05c87ac4dbad84cbd4 2024-11-22T03:41:23,358 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=90 updating hbase:meta row=47b13a1c78111d05c87ac4dbad84cbd4, regionState=CLOSED 2024-11-22T03:41:23,359 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=91, ppid=90, state=RUNNABLE, hasLock=false; CloseRegionProcedure 47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 because future has completed 2024-11-22T03:41:23,361 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=91, resume processing ppid=90 2024-11-22T03:41:23,361 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=91, ppid=90, state=SUCCESS, hasLock=false; CloseRegionProcedure 47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 in 164 msec 2024-11-22T03:41:23,362 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=90, ppid=89, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=47b13a1c78111d05c87ac4dbad84cbd4, REOPEN/MOVE; state=CLOSED, location=c85114ed5096,42545,1732246786482; forceNewPlan=false, retain=true 2024-11-22T03:41:23,513 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=90 updating hbase:meta row=47b13a1c78111d05c87ac4dbad84cbd4, regionState=OPENING, regionLocation=c85114ed5096,42545,1732246786482 2024-11-22T03:41:23,515 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=90, ppid=89, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=47b13a1c78111d05c87ac4dbad84cbd4, REOPEN/MOVE because future has completed 2024-11-22T03:41:23,516 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=92, ppid=90, state=RUNNABLE, hasLock=false; OpenRegionProcedure 47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482}] 2024-11-22T03:41:23,672 INFO [RS_OPEN_REGION-regionserver/c85114ed5096:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4. 2024-11-22T03:41:23,672 DEBUG [RS_OPEN_REGION-regionserver/c85114ed5096:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(7752): Opening region: {ENCODED => 47b13a1c78111d05c87ac4dbad84cbd4, NAME => 'TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4.', STARTKEY => '', ENDKEY => ''} 2024-11-22T03:41:23,673 DEBUG [RS_OPEN_REGION-regionserver/c85114ed5096:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 47b13a1c78111d05c87ac4dbad84cbd4 2024-11-22T03:41:23,673 DEBUG [RS_OPEN_REGION-regionserver/c85114ed5096:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-22T03:41:23,673 DEBUG [RS_OPEN_REGION-regionserver/c85114ed5096:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(7794): checking encryption for 47b13a1c78111d05c87ac4dbad84cbd4 2024-11-22T03:41:23,673 DEBUG [RS_OPEN_REGION-regionserver/c85114ed5096:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(7797): checking classloading for 47b13a1c78111d05c87ac4dbad84cbd4 2024-11-22T03:41:23,675 INFO [StoreOpener-47b13a1c78111d05c87ac4dbad84cbd4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 47b13a1c78111d05c87ac4dbad84cbd4 2024-11-22T03:41:23,676 INFO [StoreOpener-47b13a1c78111d05c87ac4dbad84cbd4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 47b13a1c78111d05c87ac4dbad84cbd4 columnFamilyName A 2024-11-22T03:41:23,677 DEBUG [StoreOpener-47b13a1c78111d05c87ac4dbad84cbd4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:23,678 INFO [StoreOpener-47b13a1c78111d05c87ac4dbad84cbd4-1 {}] regionserver.HStore(327): Store=47b13a1c78111d05c87ac4dbad84cbd4/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-22T03:41:23,679 INFO [StoreOpener-47b13a1c78111d05c87ac4dbad84cbd4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 47b13a1c78111d05c87ac4dbad84cbd4 2024-11-22T03:41:23,679 INFO [StoreOpener-47b13a1c78111d05c87ac4dbad84cbd4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 47b13a1c78111d05c87ac4dbad84cbd4 columnFamilyName B 2024-11-22T03:41:23,680 DEBUG [StoreOpener-47b13a1c78111d05c87ac4dbad84cbd4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:23,680 INFO [StoreOpener-47b13a1c78111d05c87ac4dbad84cbd4-1 {}] regionserver.HStore(327): Store=47b13a1c78111d05c87ac4dbad84cbd4/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-22T03:41:23,680 INFO [StoreOpener-47b13a1c78111d05c87ac4dbad84cbd4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 47b13a1c78111d05c87ac4dbad84cbd4 2024-11-22T03:41:23,681 INFO [StoreOpener-47b13a1c78111d05c87ac4dbad84cbd4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 47b13a1c78111d05c87ac4dbad84cbd4 columnFamilyName C 2024-11-22T03:41:23,681 DEBUG [StoreOpener-47b13a1c78111d05c87ac4dbad84cbd4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:23,682 INFO [StoreOpener-47b13a1c78111d05c87ac4dbad84cbd4-1 {}] regionserver.HStore(327): Store=47b13a1c78111d05c87ac4dbad84cbd4/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-22T03:41:23,682 DEBUG [RS_OPEN_REGION-regionserver/c85114ed5096:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(1038): replaying wal for 47b13a1c78111d05c87ac4dbad84cbd4 2024-11-22T03:41:23,683 DEBUG [RS_OPEN_REGION-regionserver/c85114ed5096:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4 2024-11-22T03:41:23,684 DEBUG [RS_OPEN_REGION-regionserver/c85114ed5096:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4 2024-11-22T03:41:23,686 DEBUG [RS_OPEN_REGION-regionserver/c85114ed5096:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(1048): stopping wal replay for 47b13a1c78111d05c87ac4dbad84cbd4 2024-11-22T03:41:23,686 DEBUG [RS_OPEN_REGION-regionserver/c85114ed5096:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(1060): Cleaning up temporary data for 47b13a1c78111d05c87ac4dbad84cbd4 2024-11-22T03:41:23,686 DEBUG [RS_OPEN_REGION-regionserver/c85114ed5096:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-22T03:41:23,688 DEBUG [RS_OPEN_REGION-regionserver/c85114ed5096:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(1093): writing seq id for 47b13a1c78111d05c87ac4dbad84cbd4 2024-11-22T03:41:23,689 INFO [RS_OPEN_REGION-regionserver/c85114ed5096:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(1114): Opened 47b13a1c78111d05c87ac4dbad84cbd4; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67505521, jitterRate=0.00591064989566803}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-22T03:41:23,689 DEBUG [RS_OPEN_REGION-regionserver/c85114ed5096:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 47b13a1c78111d05c87ac4dbad84cbd4 2024-11-22T03:41:23,690 DEBUG [RS_OPEN_REGION-regionserver/c85114ed5096:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(1006): Region open journal for 47b13a1c78111d05c87ac4dbad84cbd4: Running coprocessor pre-open hook at 1732246883673Writing region info on filesystem at 1732246883673Initializing all the Stores at 1732246883674 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732246883674Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732246883675 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732246883675Cleaning up temporary data from old regions at 1732246883686 (+11 ms)Running coprocessor post-open hooks at 1732246883689 (+3 ms)Region opened successfully at 1732246883690 (+1 ms) 2024-11-22T03:41:23,691 INFO [RS_OPEN_REGION-regionserver/c85114ed5096:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., pid=92, masterSystemTime=1732246883668 2024-11-22T03:41:23,694 DEBUG [RS_OPEN_REGION-regionserver/c85114ed5096:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4. 2024-11-22T03:41:23,694 INFO [RS_OPEN_REGION-regionserver/c85114ed5096:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4. 2024-11-22T03:41:23,694 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=90 updating hbase:meta row=47b13a1c78111d05c87ac4dbad84cbd4, regionState=OPEN, openSeqNum=5, regionLocation=c85114ed5096,42545,1732246786482 2024-11-22T03:41:23,696 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=92, ppid=90, state=RUNNABLE, hasLock=false; OpenRegionProcedure 47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 because future has completed 2024-11-22T03:41:23,701 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=92, resume processing ppid=90 2024-11-22T03:41:23,701 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=92, ppid=90, state=SUCCESS, hasLock=false; OpenRegionProcedure 47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 in 182 msec 2024-11-22T03:41:23,702 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=90, resume processing ppid=89 2024-11-22T03:41:23,702 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=90, ppid=89, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=47b13a1c78111d05c87ac4dbad84cbd4, REOPEN/MOVE in 509 msec 2024-11-22T03:41:23,705 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=89, resume processing ppid=88 2024-11-22T03:41:23,705 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=89, ppid=88, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 513 msec 2024-11-22T03:41:23,707 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=88, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 939 msec 2024-11-22T03:41:23,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=88 2024-11-22T03:41:23,708 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-22T03:41:23,708 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1cf7f63f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T03:41:23,708 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c85114ed5096,42627,-1 for getting cluster id 2024-11-22T03:41:23,709 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-22T03:41:23,709 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '67035b7f-c649-44a8-9db7-17adff49cedb' 2024-11-22T03:41:23,710 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-22T03:41:23,710 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "67035b7f-c649-44a8-9db7-17adff49cedb" 2024-11-22T03:41:23,710 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2fbf706e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T03:41:23,710 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c85114ed5096,42627,-1] 2024-11-22T03:41:23,710 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-22T03:41:23,710 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T03:41:23,711 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53542, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-22T03:41:23,712 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7dbfa489, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T03:41:23,712 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4600746, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T03:41:23,712 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c85114ed5096,42627,-1 for getting cluster id 2024-11-22T03:41:23,712 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-22T03:41:23,713 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '67035b7f-c649-44a8-9db7-17adff49cedb' 2024-11-22T03:41:23,713 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-22T03:41:23,713 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "67035b7f-c649-44a8-9db7-17adff49cedb" 2024-11-22T03:41:23,713 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4bc68e6e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T03:41:23,713 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c85114ed5096,42627,-1] 2024-11-22T03:41:23,714 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-22T03:41:23,714 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T03:41:23,715 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53548, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-22T03:41:23,716 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@442cfb9f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T03:41:23,716 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6447def3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T03:41:23,716 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c85114ed5096,42627,-1 for getting cluster id 2024-11-22T03:41:23,717 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-22T03:41:23,717 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '67035b7f-c649-44a8-9db7-17adff49cedb' 2024-11-22T03:41:23,717 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-22T03:41:23,718 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "67035b7f-c649-44a8-9db7-17adff49cedb" 2024-11-22T03:41:23,718 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@25a762e8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T03:41:23,718 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c85114ed5096,42627,-1] 2024-11-22T03:41:23,718 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-22T03:41:23,718 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T03:41:23,719 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53552, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-22T03:41:23,719 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4d9ed662, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T03:41:23,720 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5345722e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T03:41:23,720 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c85114ed5096,42627,-1 for getting cluster id 2024-11-22T03:41:23,720 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-22T03:41:23,720 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '67035b7f-c649-44a8-9db7-17adff49cedb' 2024-11-22T03:41:23,720 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-22T03:41:23,720 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "67035b7f-c649-44a8-9db7-17adff49cedb" 2024-11-22T03:41:23,720 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@e64c13d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T03:41:23,720 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c85114ed5096,42627,-1] 2024-11-22T03:41:23,721 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-22T03:41:23,721 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T03:41:23,721 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53568, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-22T03:41:23,722 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@10be8c2e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T03:41:23,722 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4dd3a90f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T03:41:23,722 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c85114ed5096,42627,-1 for getting cluster id 2024-11-22T03:41:23,722 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-22T03:41:23,723 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '67035b7f-c649-44a8-9db7-17adff49cedb' 2024-11-22T03:41:23,723 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-22T03:41:23,723 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "67035b7f-c649-44a8-9db7-17adff49cedb" 2024-11-22T03:41:23,723 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7e43bf85, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T03:41:23,723 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c85114ed5096,42627,-1] 2024-11-22T03:41:23,723 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-22T03:41:23,723 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T03:41:23,724 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53596, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-22T03:41:23,724 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@69b339a9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T03:41:23,725 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7eba3e2f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T03:41:23,725 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c85114ed5096,42627,-1 for getting cluster id 2024-11-22T03:41:23,725 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-22T03:41:23,725 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '67035b7f-c649-44a8-9db7-17adff49cedb' 2024-11-22T03:41:23,726 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-22T03:41:23,726 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "67035b7f-c649-44a8-9db7-17adff49cedb" 2024-11-22T03:41:23,726 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@647f1206, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T03:41:23,726 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c85114ed5096,42627,-1] 2024-11-22T03:41:23,726 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-22T03:41:23,726 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T03:41:23,727 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53616, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-22T03:41:23,727 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5f3bf6ac, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T03:41:23,728 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@79570776, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T03:41:23,728 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c85114ed5096,42627,-1 for getting cluster id 2024-11-22T03:41:23,728 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-22T03:41:23,728 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '67035b7f-c649-44a8-9db7-17adff49cedb' 2024-11-22T03:41:23,728 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-22T03:41:23,728 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "67035b7f-c649-44a8-9db7-17adff49cedb" 2024-11-22T03:41:23,728 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5c92336c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T03:41:23,728 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c85114ed5096,42627,-1] 2024-11-22T03:41:23,729 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-22T03:41:23,729 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T03:41:23,729 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53648, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-22T03:41:23,730 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@57eff370, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T03:41:23,730 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@43065a6b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T03:41:23,730 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c85114ed5096,42627,-1 for getting cluster id 2024-11-22T03:41:23,730 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-22T03:41:23,731 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '67035b7f-c649-44a8-9db7-17adff49cedb' 2024-11-22T03:41:23,731 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-22T03:41:23,731 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "67035b7f-c649-44a8-9db7-17adff49cedb" 2024-11-22T03:41:23,731 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@411d2080, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T03:41:23,731 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c85114ed5096,42627,-1] 2024-11-22T03:41:23,731 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-22T03:41:23,732 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T03:41:23,732 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53668, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-22T03:41:23,732 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@41609696, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T03:41:23,733 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@24bcd92c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T03:41:23,733 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c85114ed5096,42627,-1 for getting cluster id 2024-11-22T03:41:23,733 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-22T03:41:23,734 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '67035b7f-c649-44a8-9db7-17adff49cedb' 2024-11-22T03:41:23,734 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-22T03:41:23,734 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "67035b7f-c649-44a8-9db7-17adff49cedb" 2024-11-22T03:41:23,734 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6a00daa4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T03:41:23,734 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c85114ed5096,42627,-1] 2024-11-22T03:41:23,734 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-22T03:41:23,734 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T03:41:23,735 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53682, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-22T03:41:23,735 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7870b537, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T03:41:23,736 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4b779047, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T03:41:23,736 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c85114ed5096,42627,-1 for getting cluster id 2024-11-22T03:41:23,736 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-22T03:41:23,736 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '67035b7f-c649-44a8-9db7-17adff49cedb' 2024-11-22T03:41:23,736 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-22T03:41:23,737 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "67035b7f-c649-44a8-9db7-17adff49cedb" 2024-11-22T03:41:23,737 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3922ace3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T03:41:23,737 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c85114ed5096,42627,-1] 2024-11-22T03:41:23,737 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-22T03:41:23,737 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T03:41:23,737 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53696, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-22T03:41:23,738 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6db86d6d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T03:41:23,738 DEBUG [Thread-1371 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-22T03:41:23,738 DEBUG [Thread-1382 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-22T03:41:23,738 DEBUG [Thread-1384 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-22T03:41:23,738 DEBUG [Thread-1373 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-22T03:41:23,739 DEBUG [Thread-1376 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-22T03:41:23,739 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42627 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-22T03:41:23,739 DEBUG [Thread-1378 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-22T03:41:23,739 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c85114ed5096,42545,1732246786482, seqNum=-1] 2024-11-22T03:41:23,739 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c85114ed5096,42545,1732246786482, seqNum=-1] 2024-11-22T03:41:23,739 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c85114ed5096,42545,1732246786482, seqNum=-1] 2024-11-22T03:41:23,739 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c85114ed5096,42545,1732246786482, seqNum=-1] 2024-11-22T03:41:23,739 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-22T03:41:23,739 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-22T03:41:23,740 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-22T03:41:23,740 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c85114ed5096,42545,1732246786482, seqNum=-1] 2024-11-22T03:41:23,740 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-22T03:41:23,740 DEBUG [Thread-1380 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-22T03:41:23,740 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c85114ed5096,42545,1732246786482, seqNum=-1] 2024-11-22T03:41:23,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42627 {}] procedure2.ProcedureExecutor(1139): Stored pid=93, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=93, table=TestAcidGuarantees 2024-11-22T03:41:23,740 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-22T03:41:23,740 DEBUG [Thread-1369 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-22T03:41:23,740 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-22T03:41:23,741 DEBUG [Thread-1365 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-22T03:41:23,741 DEBUG [Thread-1380 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c85114ed5096,42545,1732246786482, seqNum=-1] 2024-11-22T03:41:23,741 DEBUG [Thread-1367 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-22T03:41:23,741 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52414, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-22T03:41:23,741 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52418, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-22T03:41:23,741 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c85114ed5096,42545,1732246786482, seqNum=-1] 2024-11-22T03:41:23,741 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c85114ed5096,42545,1732246786482, seqNum=-1] 2024-11-22T03:41:23,741 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52452, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-22T03:41:23,741 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52458, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-22T03:41:23,742 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-22T03:41:23,742 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c85114ed5096,42545,1732246786482, seqNum=-1] 2024-11-22T03:41:23,742 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-22T03:41:23,742 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-22T03:41:23,742 DEBUG [Thread-1380 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-22T03:41:23,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-22T03:41:23,742 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52416, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-22T03:41:23,743 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52486, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-22T03:41:23,743 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52478, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-22T03:41:23,743 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52472, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-22T03:41:23,743 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=93, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=93, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-22T03:41:23,743 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52494, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-22T03:41:23,743 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52510, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-22T03:41:23,743 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5] 2024-11-22T03:41:23,744 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=93, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=93, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-22T03:41:23,744 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=94, ppid=93, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-22T03:41:23,744 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5] 2024-11-22T03:41:23,745 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5] 2024-11-22T03:41:23,745 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5] 2024-11-22T03:41:23,746 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5] 2024-11-22T03:41:23,746 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5] 2024-11-22T03:41:23,748 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5] 2024-11-22T03:41:23,748 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5] 2024-11-22T03:41:23,749 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5] 2024-11-22T03:41:23,750 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5] 2024-11-22T03:41:23,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(8855): Flush requested on 47b13a1c78111d05c87ac4dbad84cbd4 2024-11-22T03:41:23,757 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-22T03:41:23,758 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 47b13a1c78111d05c87ac4dbad84cbd4 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-22T03:41:23,787 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411226165be01e57a4cd7a6e232f6c41ea137_47b13a1c78111d05c87ac4dbad84cbd4 is 50, key is test_row_0/A:col10/1732246883753/Put/seqid=0 2024-11-22T03:41:23,797 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742118_1294 (size=14594) 2024-11-22T03:41:23,808 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:23,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52458 deadline: 1732246943803, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:23,808 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:23,808 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:23,808 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:41:23,814 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:23,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52478 deadline: 1732246943805, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:23,815 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:23,815 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:23,815 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:41:23,819 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:23,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52486 deadline: 1732246943806, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:23,819 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:23,819 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:23,819 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:23,819 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:41:23,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52414 deadline: 1732246943807, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:23,820 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:23,819 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:23,820 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:23,820 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:41:23,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52472 deadline: 1732246943808, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:23,820 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:23,820 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:23,820 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:41:23,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-22T03:41:23,899 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42545 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-22T03:41:23,899 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4. 2024-11-22T03:41:23,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4. as already flushing 2024-11-22T03:41:23,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4. 2024-11-22T03:41:23,900 ERROR [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => 47b13a1c78111d05c87ac4dbad84cbd4, NAME => 'TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:41:23,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => 47b13a1c78111d05c87ac4dbad84cbd4, NAME => 'TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:41:23,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42627 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 47b13a1c78111d05c87ac4dbad84cbd4, NAME => 'TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 47b13a1c78111d05c87ac4dbad84cbd4, NAME => 'TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:41:23,924 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:23,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52458 deadline: 1732246943919, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:23,924 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:23,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52478 deadline: 1732246943919, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:23,924 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:23,924 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:23,924 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:41:23,924 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:23,924 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:23,924 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:41:23,932 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:23,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52486 deadline: 1732246943929, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:23,933 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:23,933 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:23,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52414 deadline: 1732246943929, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:23,933 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:23,933 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:41:23,933 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:23,933 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:23,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52472 deadline: 1732246943929, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:23,933 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:23,933 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:41:23,933 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:23,933 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:23,933 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:41:24,052 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42545 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-22T03:41:24,052 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4. 2024-11-22T03:41:24,052 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4. as already flushing 2024-11-22T03:41:24,052 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4. 2024-11-22T03:41:24,052 ERROR [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => 47b13a1c78111d05c87ac4dbad84cbd4, NAME => 'TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:41:24,052 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => 47b13a1c78111d05c87ac4dbad84cbd4, NAME => 'TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:41:24,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42627 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 47b13a1c78111d05c87ac4dbad84cbd4, NAME => 'TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 47b13a1c78111d05c87ac4dbad84cbd4, NAME => 'TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:41:24,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-22T03:41:24,132 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:24,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52458 deadline: 1732246944129, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:24,133 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:24,133 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:24,133 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:24,133 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:41:24,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52478 deadline: 1732246944129, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:24,133 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:24,133 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:24,133 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:41:24,142 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:24,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52486 deadline: 1732246944138, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:24,142 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:24,142 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:24,142 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:24,142 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:41:24,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52472 deadline: 1732246944139, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:24,143 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:24,143 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:24,143 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:24,143 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:41:24,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52414 deadline: 1732246944139, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:24,143 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:24,143 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:24,143 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:41:24,198 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:24,203 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411226165be01e57a4cd7a6e232f6c41ea137_47b13a1c78111d05c87ac4dbad84cbd4 to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411226165be01e57a4cd7a6e232f6c41ea137_47b13a1c78111d05c87ac4dbad84cbd4 2024-11-22T03:41:24,203 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/.tmp/A/0451467100304ffdaa99be2cab971cb0, store: [table=TestAcidGuarantees family=A region=47b13a1c78111d05c87ac4dbad84cbd4] 2024-11-22T03:41:24,204 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/.tmp/A/0451467100304ffdaa99be2cab971cb0 is 175, key is test_row_0/A:col10/1732246883753/Put/seqid=0 2024-11-22T03:41:24,205 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42545 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-22T03:41:24,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4. 2024-11-22T03:41:24,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4. as already flushing 2024-11-22T03:41:24,206 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4. 2024-11-22T03:41:24,206 ERROR [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => 47b13a1c78111d05c87ac4dbad84cbd4, NAME => 'TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:41:24,206 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => 47b13a1c78111d05c87ac4dbad84cbd4, NAME => 'TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:41:24,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42627 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 47b13a1c78111d05c87ac4dbad84cbd4, NAME => 'TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 47b13a1c78111d05c87ac4dbad84cbd4, NAME => 'TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:41:24,207 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742119_1295 (size=39549) 2024-11-22T03:41:24,358 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42545 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-22T03:41:24,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4. 2024-11-22T03:41:24,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4. as already flushing 2024-11-22T03:41:24,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4. 2024-11-22T03:41:24,359 ERROR [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => 47b13a1c78111d05c87ac4dbad84cbd4, NAME => 'TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:41:24,359 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => 47b13a1c78111d05c87ac4dbad84cbd4, NAME => 'TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:41:24,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42627 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 47b13a1c78111d05c87ac4dbad84cbd4, NAME => 'TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 47b13a1c78111d05c87ac4dbad84cbd4, NAME => 'TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:41:24,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-22T03:41:24,443 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:24,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52458 deadline: 1732246944439, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:24,443 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:24,443 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:24,444 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:24,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52478 deadline: 1732246944439, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:24,444 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:41:24,444 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:24,444 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:24,444 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:41:24,453 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:24,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52486 deadline: 1732246944448, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:24,453 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:24,453 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:24,453 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:24,453 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:41:24,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52472 deadline: 1732246944448, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:24,453 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:24,453 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:24,453 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:41:24,453 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:24,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52414 deadline: 1732246944448, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:24,454 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:24,454 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:24,454 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:41:24,511 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42545 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-22T03:41:24,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4. 2024-11-22T03:41:24,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4. as already flushing 2024-11-22T03:41:24,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4. 2024-11-22T03:41:24,511 ERROR [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => 47b13a1c78111d05c87ac4dbad84cbd4, NAME => 'TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:41:24,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => 47b13a1c78111d05c87ac4dbad84cbd4, NAME => 'TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:41:24,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42627 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 47b13a1c78111d05c87ac4dbad84cbd4, NAME => 'TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 47b13a1c78111d05c87ac4dbad84cbd4, NAME => 'TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:41:24,608 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=15, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/.tmp/A/0451467100304ffdaa99be2cab971cb0 2024-11-22T03:41:24,626 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/.tmp/B/fcda2d06b5b1432c8d72e01dcf2d3980 is 50, key is test_row_0/B:col10/1732246883753/Put/seqid=0 2024-11-22T03:41:24,629 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742120_1296 (size=12001) 2024-11-22T03:41:24,664 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42545 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-22T03:41:24,664 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4. 2024-11-22T03:41:24,664 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4. as already flushing 2024-11-22T03:41:24,664 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4. 2024-11-22T03:41:24,664 ERROR [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => 47b13a1c78111d05c87ac4dbad84cbd4, NAME => 'TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:41:24,664 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => 47b13a1c78111d05c87ac4dbad84cbd4, NAME => 'TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:41:24,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42627 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 47b13a1c78111d05c87ac4dbad84cbd4, NAME => 'TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 47b13a1c78111d05c87ac4dbad84cbd4, NAME => 'TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:41:24,815 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42545 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-22T03:41:24,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4. 2024-11-22T03:41:24,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4. as already flushing 2024-11-22T03:41:24,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4. 2024-11-22T03:41:24,816 ERROR [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => 47b13a1c78111d05c87ac4dbad84cbd4, NAME => 'TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:41:24,816 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => 47b13a1c78111d05c87ac4dbad84cbd4, NAME => 'TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:41:24,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42627 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 47b13a1c78111d05c87ac4dbad84cbd4, NAME => 'TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 47b13a1c78111d05c87ac4dbad84cbd4, NAME => 'TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:41:24,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-22T03:41:24,953 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:24,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52458 deadline: 1732246944948, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:24,953 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:24,953 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:24,953 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:24,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52478 deadline: 1732246944948, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:24,953 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:41:24,953 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:24,954 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:24,954 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:41:24,962 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:24,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52486 deadline: 1732246944958, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:24,962 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:24,962 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:24,962 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:24,962 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:41:24,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52472 deadline: 1732246944958, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:24,962 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:24,962 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:24,962 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:41:24,968 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42545 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-22T03:41:24,968 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4. 2024-11-22T03:41:24,968 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4. as already flushing 2024-11-22T03:41:24,968 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4. 2024-11-22T03:41:24,968 ERROR [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => 47b13a1c78111d05c87ac4dbad84cbd4, NAME => 'TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:41:24,968 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => 47b13a1c78111d05c87ac4dbad84cbd4, NAME => 'TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:41:24,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42627 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 47b13a1c78111d05c87ac4dbad84cbd4, NAME => 'TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 47b13a1c78111d05c87ac4dbad84cbd4, NAME => 'TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:41:24,973 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:24,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52414 deadline: 1732246944969, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:24,973 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:24,973 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:24,973 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:41:25,029 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/.tmp/B/fcda2d06b5b1432c8d72e01dcf2d3980 2024-11-22T03:41:25,052 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/.tmp/C/ebb98f359c0c4eff9cb9b896c25d2092 is 50, key is test_row_0/C:col10/1732246883753/Put/seqid=0 2024-11-22T03:41:25,055 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742121_1297 (size=12001) 2024-11-22T03:41:25,120 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42545 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-22T03:41:25,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4. 2024-11-22T03:41:25,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4. as already flushing 2024-11-22T03:41:25,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4. 2024-11-22T03:41:25,120 ERROR [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => 47b13a1c78111d05c87ac4dbad84cbd4, NAME => 'TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:41:25,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => 47b13a1c78111d05c87ac4dbad84cbd4, NAME => 'TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:41:25,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42627 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 47b13a1c78111d05c87ac4dbad84cbd4, NAME => 'TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 47b13a1c78111d05c87ac4dbad84cbd4, NAME => 'TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:41:25,273 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42545 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-22T03:41:25,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4. 2024-11-22T03:41:25,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4. as already flushing 2024-11-22T03:41:25,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4. 2024-11-22T03:41:25,273 ERROR [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => 47b13a1c78111d05c87ac4dbad84cbd4, NAME => 'TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:41:25,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => 47b13a1c78111d05c87ac4dbad84cbd4, NAME => 'TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:41:25,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42627 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 47b13a1c78111d05c87ac4dbad84cbd4, NAME => 'TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 47b13a1c78111d05c87ac4dbad84cbd4, NAME => 'TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:41:25,425 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42545 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-22T03:41:25,426 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4. 2024-11-22T03:41:25,426 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4. as already flushing 2024-11-22T03:41:25,426 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4. 2024-11-22T03:41:25,426 ERROR [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => 47b13a1c78111d05c87ac4dbad84cbd4, NAME => 'TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:41:25,426 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => 47b13a1c78111d05c87ac4dbad84cbd4, NAME => 'TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:41:25,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42627 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 47b13a1c78111d05c87ac4dbad84cbd4, NAME => 'TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 47b13a1c78111d05c87ac4dbad84cbd4, NAME => 'TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:41:25,455 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/.tmp/C/ebb98f359c0c4eff9cb9b896c25d2092 2024-11-22T03:41:25,459 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/.tmp/A/0451467100304ffdaa99be2cab971cb0 as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/A/0451467100304ffdaa99be2cab971cb0 2024-11-22T03:41:25,463 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/A/0451467100304ffdaa99be2cab971cb0, entries=200, sequenceid=15, filesize=38.6 K 2024-11-22T03:41:25,464 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/.tmp/B/fcda2d06b5b1432c8d72e01dcf2d3980 as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/B/fcda2d06b5b1432c8d72e01dcf2d3980 2024-11-22T03:41:25,467 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/B/fcda2d06b5b1432c8d72e01dcf2d3980, entries=150, sequenceid=15, filesize=11.7 K 2024-11-22T03:41:25,467 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/.tmp/C/ebb98f359c0c4eff9cb9b896c25d2092 as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/C/ebb98f359c0c4eff9cb9b896c25d2092 2024-11-22T03:41:25,471 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/C/ebb98f359c0c4eff9cb9b896c25d2092, entries=150, sequenceid=15, filesize=11.7 K 2024-11-22T03:41:25,471 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 47b13a1c78111d05c87ac4dbad84cbd4 in 1714ms, sequenceid=15, compaction requested=false 2024-11-22T03:41:25,471 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 47b13a1c78111d05c87ac4dbad84cbd4: 2024-11-22T03:41:25,578 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42545 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-22T03:41:25,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4. 2024-11-22T03:41:25,578 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2902): Flushing 47b13a1c78111d05c87ac4dbad84cbd4 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-22T03:41:25,584 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241122f157dbfd6b8243a7bd204bf9a4a915cb_47b13a1c78111d05c87ac4dbad84cbd4 is 50, key is test_row_0/A:col10/1732246883807/Put/seqid=0 2024-11-22T03:41:25,587 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742122_1298 (size=12154) 2024-11-22T03:41:25,758 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-22T03:41:25,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-22T03:41:25,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(8855): Flush requested on 47b13a1c78111d05c87ac4dbad84cbd4 2024-11-22T03:41:25,963 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4. as already flushing 2024-11-22T03:41:25,987 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:25,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52472 deadline: 1732246945979, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:25,988 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:25,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52478 deadline: 1732246945981, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:25,988 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to c85114ed5096,42545,1732246786482 for 'test_row_2' in 47b13a1c78111d05c87ac4dbad84cbd4 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2179 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-22T03:41:25,988 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:25,988 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:25,988 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:41:25,989 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:25,989 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:25,989 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:25,989 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:41:25,991 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:25,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52458 deadline: 1732246945983, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:25,991 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:25,991 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:25,991 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:41:25,992 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:25,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52486 deadline: 1732246945987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:25,992 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:25,992 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:25,992 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:41:25,993 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241122f157dbfd6b8243a7bd204bf9a4a915cb_47b13a1c78111d05c87ac4dbad84cbd4 to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241122f157dbfd6b8243a7bd204bf9a4a915cb_47b13a1c78111d05c87ac4dbad84cbd4 2024-11-22T03:41:25,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/.tmp/A/be8a456ced4d4c2da3441d8fa372a153, store: [table=TestAcidGuarantees family=A region=47b13a1c78111d05c87ac4dbad84cbd4] 2024-11-22T03:41:25,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/.tmp/A/be8a456ced4d4c2da3441d8fa372a153 is 175, key is test_row_0/A:col10/1732246883807/Put/seqid=0 2024-11-22T03:41:25,998 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:25,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52414 deadline: 1732246945989, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:25,999 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to c85114ed5096,42545,1732246786482 for 'test_row_0' in 47b13a1c78111d05c87ac4dbad84cbd4 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2191 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-22T03:41:25,999 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:25,999 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:25,999 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:41:26,004 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742123_1299 (size=30955) 2024-11-22T03:41:26,004 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=40, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/.tmp/A/be8a456ced4d4c2da3441d8fa372a153 2024-11-22T03:41:26,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/.tmp/B/66033de3255845928fb1a0fd8a15ba8a is 50, key is test_row_0/B:col10/1732246883807/Put/seqid=0 2024-11-22T03:41:26,027 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742124_1300 (size=12001) 2024-11-22T03:41:26,104 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:26,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52478 deadline: 1732246946099, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:26,104 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:26,104 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:26,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52458 deadline: 1732246946099, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:26,104 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:26,104 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:41:26,105 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:26,105 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:26,105 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:26,105 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:41:26,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52486 deadline: 1732246946099, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:26,105 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:26,105 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:26,105 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:41:26,310 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:26,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52478 deadline: 1732246946309, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:26,310 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:26,310 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:26,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52458 deadline: 1732246946309, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:26,310 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:26,310 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:41:26,310 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:26,310 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:26,310 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:26,311 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:41:26,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52486 deadline: 1732246946309, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:26,311 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:26,311 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:26,311 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:41:26,428 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=40 (bloomFilter=true), to=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/.tmp/B/66033de3255845928fb1a0fd8a15ba8a 2024-11-22T03:41:26,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/.tmp/C/d982949954404b3383facec71d14724d is 50, key is test_row_0/C:col10/1732246883807/Put/seqid=0 2024-11-22T03:41:26,438 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742125_1301 (size=12001) 2024-11-22T03:41:26,625 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:26,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52478 deadline: 1732246946618, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:26,626 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:26,626 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:26,626 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:26,626 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:41:26,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52486 deadline: 1732246946618, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:26,626 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:26,626 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:26,626 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:26,626 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:41:26,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52458 deadline: 1732246946619, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:26,627 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:26,627 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:26,627 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:41:26,839 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=40 (bloomFilter=true), to=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/.tmp/C/d982949954404b3383facec71d14724d 2024-11-22T03:41:26,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/.tmp/A/be8a456ced4d4c2da3441d8fa372a153 as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/A/be8a456ced4d4c2da3441d8fa372a153 2024-11-22T03:41:26,845 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/A/be8a456ced4d4c2da3441d8fa372a153, entries=150, sequenceid=40, filesize=30.2 K 2024-11-22T03:41:26,846 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/.tmp/B/66033de3255845928fb1a0fd8a15ba8a as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/B/66033de3255845928fb1a0fd8a15ba8a 2024-11-22T03:41:26,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,849 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/B/66033de3255845928fb1a0fd8a15ba8a, entries=150, sequenceid=40, filesize=11.7 K 2024-11-22T03:41:26,850 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/.tmp/C/d982949954404b3383facec71d14724d as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/C/d982949954404b3383facec71d14724d 2024-11-22T03:41:26,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,854 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/C/d982949954404b3383facec71d14724d, entries=150, sequenceid=40, filesize=11.7 K 2024-11-22T03:41:26,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,855 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 47b13a1c78111d05c87ac4dbad84cbd4 in 1276ms, sequenceid=40, compaction requested=false 2024-11-22T03:41:26,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2603): Flush status journal for 47b13a1c78111d05c87ac4dbad84cbd4: 2024-11-22T03:41:26,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4. 2024-11-22T03:41:26,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=94 2024-11-22T03:41:26,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42627 {}] master.HMaster(4169): Remote procedure done, pid=94 2024-11-22T03:41:26,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,858 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=94, resume processing ppid=93 2024-11-22T03:41:26,858 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=94, ppid=93, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.1120 sec 2024-11-22T03:41:26,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,859 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=93, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=93, table=TestAcidGuarantees in 3.1190 sec 2024-11-22T03:41:26,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:26,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(8855): Flush requested on 47b13a1c78111d05c87ac4dbad84cbd4 2024-11-22T03:41:27,148 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-22T03:41:27,149 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 47b13a1c78111d05c87ac4dbad84cbd4 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-22T03:41:27,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,157 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241122f20d3251941948dab91f6c2076aca01c_47b13a1c78111d05c87ac4dbad84cbd4 is 50, key is test_row_0/A:col10/1732246885986/Put/seqid=0 2024-11-22T03:41:27,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,178 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742126_1302 (size=12154) 2024-11-22T03:41:27,258 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:27,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52478 deadline: 1732246947249, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:27,259 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:27,259 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:27,259 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:41:27,267 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:27,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52486 deadline: 1732246947259, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:27,268 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:27,268 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:27,268 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:27,268 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:41:27,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52458 deadline: 1732246947259, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:27,268 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:27,268 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:27,268 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:41:27,371 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:27,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52478 deadline: 1732246947368, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:27,371 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:27,371 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:27,371 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:41:27,385 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:27,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52486 deadline: 1732246947379, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:27,385 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:27,385 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:27,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52458 deadline: 1732246947379, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:27,385 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:27,385 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:41:27,386 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:27,386 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:27,386 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:41:27,579 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:27,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52478 deadline: 1732246947578, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:27,579 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:27,579 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:27,579 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:27,579 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:41:27,583 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241122f20d3251941948dab91f6c2076aca01c_47b13a1c78111d05c87ac4dbad84cbd4 to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241122f20d3251941948dab91f6c2076aca01c_47b13a1c78111d05c87ac4dbad84cbd4 2024-11-22T03:41:27,584 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/.tmp/A/c2675a6a2c304fd98adb6a9713f9e799, store: [table=TestAcidGuarantees family=A region=47b13a1c78111d05c87ac4dbad84cbd4] 2024-11-22T03:41:27,585 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/.tmp/A/c2675a6a2c304fd98adb6a9713f9e799 is 175, key is test_row_0/A:col10/1732246885986/Put/seqid=0 2024-11-22T03:41:27,594 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:27,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52486 deadline: 1732246947589, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:27,594 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:27,594 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:27,594 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:27,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52458 deadline: 1732246947589, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:27,594 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:41:27,594 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:27,594 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:27,595 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:41:27,597 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742127_1303 (size=30955) 2024-11-22T03:41:27,894 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:27,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52478 deadline: 1732246947888, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:27,895 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:27,895 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:27,895 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:41:27,906 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:27,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52486 deadline: 1732246947899, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:27,906 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:27,906 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:27,906 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:27,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52458 deadline: 1732246947899, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:27,906 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:41:27,907 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:27,907 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:27,907 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:41:27,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-22T03:41:27,918 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-22T03:41:27,919 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42627 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-22T03:41:27,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42627 {}] procedure2.ProcedureExecutor(1139): Stored pid=95, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees 2024-11-22T03:41:27,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-22T03:41:27,921 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=95, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-22T03:41:27,921 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=95, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-22T03:41:27,922 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=96, ppid=95, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-22T03:41:27,998 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=52, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/.tmp/A/c2675a6a2c304fd98adb6a9713f9e799 2024-11-22T03:41:28,003 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/.tmp/B/e3ca809790034c49b0d925a373e412ff is 50, key is test_row_0/B:col10/1732246885986/Put/seqid=0 2024-11-22T03:41:28,006 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742128_1304 (size=12001) 2024-11-22T03:41:28,014 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:28,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52472 deadline: 1732246948009, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:28,015 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:28,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52414 deadline: 1732246948009, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:28,015 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to c85114ed5096,42545,1732246786482 for 'test_row_0' in 47b13a1c78111d05c87ac4dbad84cbd4 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4208 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-22T03:41:28,015 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to c85114ed5096,42545,1732246786482 for 'test_row_2' in 47b13a1c78111d05c87ac4dbad84cbd4 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4207 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-22T03:41:28,015 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:28,015 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:28,015 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:28,015 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:28,015 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:41:28,016 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:41:28,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-22T03:41:28,073 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42545 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-22T03:41:28,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4. 2024-11-22T03:41:28,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4. as already flushing 2024-11-22T03:41:28,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4. 2024-11-22T03:41:28,074 ERROR [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 47b13a1c78111d05c87ac4dbad84cbd4, NAME => 'TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:41:28,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 47b13a1c78111d05c87ac4dbad84cbd4, NAME => 'TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:41:28,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42627 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 47b13a1c78111d05c87ac4dbad84cbd4, NAME => 'TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 47b13a1c78111d05c87ac4dbad84cbd4, NAME => 'TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:41:28,226 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42545 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-22T03:41:28,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4. 2024-11-22T03:41:28,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4. as already flushing 2024-11-22T03:41:28,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4. 2024-11-22T03:41:28,226 ERROR [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 47b13a1c78111d05c87ac4dbad84cbd4, NAME => 'TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:41:28,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 47b13a1c78111d05c87ac4dbad84cbd4, NAME => 'TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:41:28,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42627 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 47b13a1c78111d05c87ac4dbad84cbd4, NAME => 'TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 47b13a1c78111d05c87ac4dbad84cbd4, NAME => 'TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:41:28,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-22T03:41:28,378 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42545 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-22T03:41:28,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4. 2024-11-22T03:41:28,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4. as already flushing 2024-11-22T03:41:28,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4. 2024-11-22T03:41:28,379 ERROR [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 47b13a1c78111d05c87ac4dbad84cbd4, NAME => 'TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:41:28,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 47b13a1c78111d05c87ac4dbad84cbd4, NAME => 'TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:41:28,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42627 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 47b13a1c78111d05c87ac4dbad84cbd4, NAME => 'TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 47b13a1c78111d05c87ac4dbad84cbd4, NAME => 'TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:41:28,403 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:28,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52478 deadline: 1732246948398, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:28,404 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:28,404 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:28,404 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:41:28,407 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=52 (bloomFilter=true), to=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/.tmp/B/e3ca809790034c49b0d925a373e412ff 2024-11-22T03:41:28,418 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:28,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52486 deadline: 1732246948409, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:28,418 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:28,418 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:28,418 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:41:28,423 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/.tmp/C/9ac4244d179443ce867842ea26f08284 is 50, key is test_row_0/C:col10/1732246885986/Put/seqid=0 2024-11-22T03:41:28,429 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:28,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52458 deadline: 1732246948419, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:28,431 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:28,431 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:28,431 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:41:28,442 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742129_1305 (size=12001) 2024-11-22T03:41:28,443 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=52 (bloomFilter=true), to=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/.tmp/C/9ac4244d179443ce867842ea26f08284 2024-11-22T03:41:28,448 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/.tmp/A/c2675a6a2c304fd98adb6a9713f9e799 as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/A/c2675a6a2c304fd98adb6a9713f9e799 2024-11-22T03:41:28,451 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/A/c2675a6a2c304fd98adb6a9713f9e799, entries=150, sequenceid=52, filesize=30.2 K 2024-11-22T03:41:28,452 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/.tmp/B/e3ca809790034c49b0d925a373e412ff as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/B/e3ca809790034c49b0d925a373e412ff 2024-11-22T03:41:28,455 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/B/e3ca809790034c49b0d925a373e412ff, entries=150, sequenceid=52, filesize=11.7 K 2024-11-22T03:41:28,456 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/.tmp/C/9ac4244d179443ce867842ea26f08284 as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/C/9ac4244d179443ce867842ea26f08284 2024-11-22T03:41:28,460 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/C/9ac4244d179443ce867842ea26f08284, entries=150, sequenceid=52, filesize=11.7 K 2024-11-22T03:41:28,461 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=147.60 KB/151140 for 47b13a1c78111d05c87ac4dbad84cbd4 in 1313ms, sequenceid=52, compaction requested=true 2024-11-22T03:41:28,461 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 47b13a1c78111d05c87ac4dbad84cbd4: 2024-11-22T03:41:28,461 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T03:41:28,461 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 47b13a1c78111d05c87ac4dbad84cbd4:A, priority=-2147483648, current under compaction store size is 1 2024-11-22T03:41:28,461 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T03:41:28,462 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T03:41:28,463 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101459 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T03:41:28,463 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.HStore(1541): 47b13a1c78111d05c87ac4dbad84cbd4/A is initiating minor compaction (all files) 2024-11-22T03:41:28,463 INFO [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 47b13a1c78111d05c87ac4dbad84cbd4/A in TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4. 2024-11-22T03:41:28,463 INFO [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/A/0451467100304ffdaa99be2cab971cb0, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/A/be8a456ced4d4c2da3441d8fa372a153, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/A/c2675a6a2c304fd98adb6a9713f9e799] into tmpdir=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/.tmp, totalSize=99.1 K 2024-11-22T03:41:28,463 INFO [RS:0;c85114ed5096:42545-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=53.85 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4. 2024-11-22T03:41:28,463 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4. files: [hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/A/0451467100304ffdaa99be2cab971cb0, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/A/be8a456ced4d4c2da3441d8fa372a153, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/A/c2675a6a2c304fd98adb6a9713f9e799] 2024-11-22T03:41:28,463 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T03:41:28,463 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HStore(1541): 47b13a1c78111d05c87ac4dbad84cbd4/B is initiating minor compaction (all files) 2024-11-22T03:41:28,464 INFO [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 47b13a1c78111d05c87ac4dbad84cbd4/B in TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4. 2024-11-22T03:41:28,464 INFO [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/B/fcda2d06b5b1432c8d72e01dcf2d3980, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/B/66033de3255845928fb1a0fd8a15ba8a, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/B/e3ca809790034c49b0d925a373e412ff] into tmpdir=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/.tmp, totalSize=35.2 K 2024-11-22T03:41:28,464 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0451467100304ffdaa99be2cab971cb0, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1732246883747 2024-11-22T03:41:28,464 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 47b13a1c78111d05c87ac4dbad84cbd4:B, priority=-2147483648, current under compaction store size is 2 2024-11-22T03:41:28,464 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] compactions.Compactor(225): Compacting fcda2d06b5b1432c8d72e01dcf2d3980, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1732246883753 2024-11-22T03:41:28,464 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T03:41:28,464 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] compactions.Compactor(225): Compacting be8a456ced4d4c2da3441d8fa372a153, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1732246883803 2024-11-22T03:41:28,464 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] compactions.Compactor(225): Compacting 66033de3255845928fb1a0fd8a15ba8a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1732246883803 2024-11-22T03:41:28,465 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] compactions.Compactor(225): Compacting c2675a6a2c304fd98adb6a9713f9e799, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1732246885980 2024-11-22T03:41:28,465 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] compactions.Compactor(225): Compacting e3ca809790034c49b0d925a373e412ff, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1732246885980 2024-11-22T03:41:28,467 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 47b13a1c78111d05c87ac4dbad84cbd4:C, priority=-2147483648, current under compaction store size is 3 2024-11-22T03:41:28,467 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T03:41:28,474 INFO [RS:0;c85114ed5096:42545-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=47b13a1c78111d05c87ac4dbad84cbd4] 2024-11-22T03:41:28,476 INFO [RS:0;c85114ed5096:42545-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 47b13a1c78111d05c87ac4dbad84cbd4#B#compaction#265 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 53.85 MB/second 2024-11-22T03:41:28,476 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/.tmp/B/7a56e285695344c1aad453081b77e8e7 is 50, key is test_row_0/B:col10/1732246885986/Put/seqid=0 2024-11-22T03:41:28,477 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024112234ee7eb11a2d4d898a71ba08a5a93a95_47b13a1c78111d05c87ac4dbad84cbd4 store=[table=TestAcidGuarantees family=A region=47b13a1c78111d05c87ac4dbad84cbd4] 2024-11-22T03:41:28,479 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024112234ee7eb11a2d4d898a71ba08a5a93a95_47b13a1c78111d05c87ac4dbad84cbd4, store=[table=TestAcidGuarantees family=A region=47b13a1c78111d05c87ac4dbad84cbd4] 2024-11-22T03:41:28,480 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112234ee7eb11a2d4d898a71ba08a5a93a95_47b13a1c78111d05c87ac4dbad84cbd4 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=47b13a1c78111d05c87ac4dbad84cbd4] 2024-11-22T03:41:28,487 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742130_1306 (size=4469) 2024-11-22T03:41:28,490 INFO [RS:0;c85114ed5096:42545-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 47b13a1c78111d05c87ac4dbad84cbd4#A#compaction#264 average throughput is 1.53 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-22T03:41:28,490 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/.tmp/A/e3be43fa335c470b8184a779503e3b29 is 175, key is test_row_0/A:col10/1732246885986/Put/seqid=0 2024-11-22T03:41:28,497 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742131_1307 (size=12104) 2024-11-22T03:41:28,503 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742132_1308 (size=31058) 2024-11-22T03:41:28,507 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/.tmp/B/7a56e285695344c1aad453081b77e8e7 as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/B/7a56e285695344c1aad453081b77e8e7 2024-11-22T03:41:28,511 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/.tmp/A/e3be43fa335c470b8184a779503e3b29 as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/A/e3be43fa335c470b8184a779503e3b29 2024-11-22T03:41:28,515 INFO [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 47b13a1c78111d05c87ac4dbad84cbd4/B of 47b13a1c78111d05c87ac4dbad84cbd4 into 7a56e285695344c1aad453081b77e8e7(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T03:41:28,515 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 47b13a1c78111d05c87ac4dbad84cbd4: 2024-11-22T03:41:28,515 INFO [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., storeName=47b13a1c78111d05c87ac4dbad84cbd4/B, priority=13, startTime=1732246888461; duration=0sec 2024-11-22T03:41:28,517 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T03:41:28,517 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 47b13a1c78111d05c87ac4dbad84cbd4:B 2024-11-22T03:41:28,517 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T03:41:28,522 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T03:41:28,522 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HStore(1541): 47b13a1c78111d05c87ac4dbad84cbd4/C is initiating minor compaction (all files) 2024-11-22T03:41:28,522 INFO [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 47b13a1c78111d05c87ac4dbad84cbd4/C in TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4. 2024-11-22T03:41:28,522 INFO [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/C/ebb98f359c0c4eff9cb9b896c25d2092, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/C/d982949954404b3383facec71d14724d, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/C/9ac4244d179443ce867842ea26f08284] into tmpdir=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/.tmp, totalSize=35.2 K 2024-11-22T03:41:28,522 INFO [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 47b13a1c78111d05c87ac4dbad84cbd4/A of 47b13a1c78111d05c87ac4dbad84cbd4 into e3be43fa335c470b8184a779503e3b29(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T03:41:28,523 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 47b13a1c78111d05c87ac4dbad84cbd4: 2024-11-22T03:41:28,523 INFO [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., storeName=47b13a1c78111d05c87ac4dbad84cbd4/A, priority=13, startTime=1732246888461; duration=0sec 2024-11-22T03:41:28,523 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T03:41:28,523 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 47b13a1c78111d05c87ac4dbad84cbd4:A 2024-11-22T03:41:28,523 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] compactions.Compactor(225): Compacting ebb98f359c0c4eff9cb9b896c25d2092, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1732246883753 2024-11-22T03:41:28,523 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] compactions.Compactor(225): Compacting d982949954404b3383facec71d14724d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1732246883803 2024-11-22T03:41:28,526 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] compactions.Compactor(225): Compacting 9ac4244d179443ce867842ea26f08284, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1732246885980 2024-11-22T03:41:28,531 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42545 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-22T03:41:28,531 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4. 2024-11-22T03:41:28,532 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2902): Flushing 47b13a1c78111d05c87ac4dbad84cbd4 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-22T03:41:28,547 INFO [RS:0;c85114ed5096:42545-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 47b13a1c78111d05c87ac4dbad84cbd4#C#compaction#266 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-22T03:41:28,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112289c0741aa1464e9b9fa277e16cd046ef_47b13a1c78111d05c87ac4dbad84cbd4 is 50, key is test_row_0/A:col10/1732246887257/Put/seqid=0 2024-11-22T03:41:28,547 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/.tmp/C/fdaf9f11c8744da0abfd74f12f387052 is 50, key is test_row_0/C:col10/1732246885986/Put/seqid=0 2024-11-22T03:41:28,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-22T03:41:28,558 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742133_1309 (size=12154) 2024-11-22T03:41:28,560 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:28,574 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112289c0741aa1464e9b9fa277e16cd046ef_47b13a1c78111d05c87ac4dbad84cbd4 to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112289c0741aa1464e9b9fa277e16cd046ef_47b13a1c78111d05c87ac4dbad84cbd4 2024-11-22T03:41:28,575 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/.tmp/A/b1584c8b0a4c49c2bfc79e77ad403d02, store: [table=TestAcidGuarantees family=A region=47b13a1c78111d05c87ac4dbad84cbd4] 2024-11-22T03:41:28,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/.tmp/A/b1584c8b0a4c49c2bfc79e77ad403d02 is 175, key is test_row_0/A:col10/1732246887257/Put/seqid=0 2024-11-22T03:41:28,586 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742134_1310 (size=12104) 2024-11-22T03:41:28,591 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/.tmp/C/fdaf9f11c8744da0abfd74f12f387052 as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/C/fdaf9f11c8744da0abfd74f12f387052 2024-11-22T03:41:28,595 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742135_1311 (size=30955) 2024-11-22T03:41:28,597 INFO [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 47b13a1c78111d05c87ac4dbad84cbd4/C of 47b13a1c78111d05c87ac4dbad84cbd4 into fdaf9f11c8744da0abfd74f12f387052(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T03:41:28,597 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 47b13a1c78111d05c87ac4dbad84cbd4: 2024-11-22T03:41:28,597 INFO [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., storeName=47b13a1c78111d05c87ac4dbad84cbd4/C, priority=13, startTime=1732246888464; duration=0sec 2024-11-22T03:41:28,597 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T03:41:28,597 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 47b13a1c78111d05c87ac4dbad84cbd4:C 2024-11-22T03:41:28,996 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=79, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/.tmp/A/b1584c8b0a4c49c2bfc79e77ad403d02 2024-11-22T03:41:29,019 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/.tmp/B/de8f517c62e6458cb317bdbea9af310a is 50, key is test_row_0/B:col10/1732246887257/Put/seqid=0 2024-11-22T03:41:29,055 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742136_1312 (size=12001) 2024-11-22T03:41:29,055 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=79 (bloomFilter=true), to=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/.tmp/B/de8f517c62e6458cb317bdbea9af310a 2024-11-22T03:41:29,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-22T03:41:29,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/.tmp/C/80ea1b6c83f94b2c96effac7620874c6 is 50, key is test_row_0/C:col10/1732246887257/Put/seqid=0 2024-11-22T03:41:29,091 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742137_1313 (size=12001) 2024-11-22T03:41:29,426 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4. as already flushing 2024-11-22T03:41:29,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(8855): Flush requested on 47b13a1c78111d05c87ac4dbad84cbd4 2024-11-22T03:41:29,467 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:29,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52478 deadline: 1732246949454, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:29,468 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:29,468 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:29,468 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:29,468 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:41:29,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52458 deadline: 1732246949454, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:29,468 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:29,468 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:29,468 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:41:29,468 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:29,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52486 deadline: 1732246949456, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:29,470 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:29,470 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:29,470 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:41:29,492 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=79 (bloomFilter=true), to=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/.tmp/C/80ea1b6c83f94b2c96effac7620874c6 2024-11-22T03:41:29,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/.tmp/A/b1584c8b0a4c49c2bfc79e77ad403d02 as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/A/b1584c8b0a4c49c2bfc79e77ad403d02 2024-11-22T03:41:29,503 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/A/b1584c8b0a4c49c2bfc79e77ad403d02, entries=150, sequenceid=79, filesize=30.2 K 2024-11-22T03:41:29,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/.tmp/B/de8f517c62e6458cb317bdbea9af310a as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/B/de8f517c62e6458cb317bdbea9af310a 2024-11-22T03:41:29,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,511 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/B/de8f517c62e6458cb317bdbea9af310a, entries=150, sequenceid=79, filesize=11.7 K 2024-11-22T03:41:29,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/.tmp/C/80ea1b6c83f94b2c96effac7620874c6 as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/C/80ea1b6c83f94b2c96effac7620874c6 2024-11-22T03:41:29,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,519 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/C/80ea1b6c83f94b2c96effac7620874c6, entries=150, sequenceid=79, filesize=11.7 K 2024-11-22T03:41:29,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,524 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 47b13a1c78111d05c87ac4dbad84cbd4 in 992ms, sequenceid=79, compaction requested=false 2024-11-22T03:41:29,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2603): Flush status journal for 47b13a1c78111d05c87ac4dbad84cbd4: 2024-11-22T03:41:29,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4. 2024-11-22T03:41:29,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=96 2024-11-22T03:41:29,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42627 {}] master.HMaster(4169): Remote procedure done, pid=96 2024-11-22T03:41:29,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,528 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=96, resume processing ppid=95 2024-11-22T03:41:29,528 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=96, ppid=95, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6030 sec 2024-11-22T03:41:29,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,531 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=95, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees in 1.6100 sec 2024-11-22T03:41:29,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,627 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-22T03:41:29,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,627 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 47b13a1c78111d05c87ac4dbad84cbd4 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-22T03:41:29,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(8855): Flush requested on 47b13a1c78111d05c87ac4dbad84cbd4 2024-11-22T03:41:29,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,655 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241122b6899f47bd6344dc9490b74d7a5b4599_47b13a1c78111d05c87ac4dbad84cbd4 is 50, key is test_row_0/A:col10/1732246889626/Put/seqid=0 2024-11-22T03:41:29,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,694 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742138_1314 (size=12154) 2024-11-22T03:41:29,695 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:29,700 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241122b6899f47bd6344dc9490b74d7a5b4599_47b13a1c78111d05c87ac4dbad84cbd4 to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241122b6899f47bd6344dc9490b74d7a5b4599_47b13a1c78111d05c87ac4dbad84cbd4 2024-11-22T03:41:29,701 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/.tmp/A/6d72541868314cb9ac09d5f073f3615f, store: [table=TestAcidGuarantees family=A region=47b13a1c78111d05c87ac4dbad84cbd4] 2024-11-22T03:41:29,702 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/.tmp/A/6d72541868314cb9ac09d5f073f3615f is 175, key is test_row_0/A:col10/1732246889626/Put/seqid=0 2024-11-22T03:41:29,743 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742139_1315 (size=30951) 2024-11-22T03:41:29,746 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=92, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/.tmp/A/6d72541868314cb9ac09d5f073f3615f 2024-11-22T03:41:29,762 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/.tmp/B/642df1c01ae243c6b2994695f3c7460d is 50, key is test_row_0/B:col10/1732246889626/Put/seqid=0 2024-11-22T03:41:29,785 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742140_1316 (size=9657) 2024-11-22T03:41:29,787 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:29,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52458 deadline: 1732246949772, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:29,787 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:29,788 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:29,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52486 deadline: 1732246949776, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:29,788 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:29,788 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:41:29,788 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:29,788 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:29,788 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:41:29,789 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:29,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52478 deadline: 1732246949777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:29,790 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:29,790 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:29,790 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:41:29,905 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:29,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52486 deadline: 1732246949899, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:29,906 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:29,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52478 deadline: 1732246949901, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:29,906 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:29,906 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:29,906 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:41:29,907 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:29,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52458 deadline: 1732246949901, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:29,907 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:29,907 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:29,907 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:29,907 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:41:29,907 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:29,907 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:41:30,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-22T03:41:30,069 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-22T03:41:30,070 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42627 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-22T03:41:30,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42627 {}] procedure2.ProcedureExecutor(1139): Stored pid=97, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees 2024-11-22T03:41:30,072 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=97, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-22T03:41:30,073 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=97, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-22T03:41:30,073 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=98, ppid=97, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-22T03:41:30,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-22T03:41:30,116 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:30,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52478 deadline: 1732246950111, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:30,118 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:30,118 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:30,118 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:41:30,123 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:30,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52486 deadline: 1732246950118, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:30,123 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:30,123 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:30,123 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:30,123 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:41:30,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52458 deadline: 1732246950119, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:30,125 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:30,125 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:30,125 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:41:30,186 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=92 (bloomFilter=true), to=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/.tmp/B/642df1c01ae243c6b2994695f3c7460d 2024-11-22T03:41:30,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-22T03:41:30,195 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/.tmp/C/5adf8b6acb714e57a81bbe590eb92096 is 50, key is test_row_0/C:col10/1732246889626/Put/seqid=0 2024-11-22T03:41:30,210 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742141_1317 (size=9657) 2024-11-22T03:41:30,211 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=92 (bloomFilter=true), to=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/.tmp/C/5adf8b6acb714e57a81bbe590eb92096 2024-11-22T03:41:30,216 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/.tmp/A/6d72541868314cb9ac09d5f073f3615f as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/A/6d72541868314cb9ac09d5f073f3615f 2024-11-22T03:41:30,226 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/A/6d72541868314cb9ac09d5f073f3615f, entries=150, sequenceid=92, filesize=30.2 K 2024-11-22T03:41:30,227 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/.tmp/B/642df1c01ae243c6b2994695f3c7460d as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/B/642df1c01ae243c6b2994695f3c7460d 2024-11-22T03:41:30,234 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42545 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-22T03:41:30,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4. 2024-11-22T03:41:30,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4. as already flushing 2024-11-22T03:41:30,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4. 2024-11-22T03:41:30,234 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/B/642df1c01ae243c6b2994695f3c7460d, entries=100, sequenceid=92, filesize=9.4 K 2024-11-22T03:41:30,234 ERROR [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 47b13a1c78111d05c87ac4dbad84cbd4, NAME => 'TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:41:30,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 47b13a1c78111d05c87ac4dbad84cbd4, NAME => 'TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:41:30,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42627 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 47b13a1c78111d05c87ac4dbad84cbd4, NAME => 'TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 47b13a1c78111d05c87ac4dbad84cbd4, NAME => 'TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:41:30,237 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/.tmp/C/5adf8b6acb714e57a81bbe590eb92096 as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/C/5adf8b6acb714e57a81bbe590eb92096 2024-11-22T03:41:30,260 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/C/5adf8b6acb714e57a81bbe590eb92096, entries=100, sequenceid=92, filesize=9.4 K 2024-11-22T03:41:30,261 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=147.60 KB/151140 for 47b13a1c78111d05c87ac4dbad84cbd4 in 634ms, sequenceid=92, compaction requested=true 2024-11-22T03:41:30,261 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 47b13a1c78111d05c87ac4dbad84cbd4: 2024-11-22T03:41:30,261 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 47b13a1c78111d05c87ac4dbad84cbd4:A, priority=-2147483648, current under compaction store size is 1 2024-11-22T03:41:30,261 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T03:41:30,261 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 47b13a1c78111d05c87ac4dbad84cbd4:B, priority=-2147483648, current under compaction store size is 2 2024-11-22T03:41:30,261 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T03:41:30,261 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T03:41:30,261 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 47b13a1c78111d05c87ac4dbad84cbd4:C, priority=-2147483648, current under compaction store size is 3 2024-11-22T03:41:30,261 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-22T03:41:30,261 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T03:41:30,263 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33762 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T03:41:30,263 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HStore(1541): 47b13a1c78111d05c87ac4dbad84cbd4/B is initiating minor compaction (all files) 2024-11-22T03:41:30,263 INFO [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 47b13a1c78111d05c87ac4dbad84cbd4/B in TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4. 2024-11-22T03:41:30,263 INFO [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/B/7a56e285695344c1aad453081b77e8e7, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/B/de8f517c62e6458cb317bdbea9af310a, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/B/642df1c01ae243c6b2994695f3c7460d] into tmpdir=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/.tmp, totalSize=33.0 K 2024-11-22T03:41:30,264 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 92964 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T03:41:30,264 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.HStore(1541): 47b13a1c78111d05c87ac4dbad84cbd4/A is initiating minor compaction (all files) 2024-11-22T03:41:30,264 INFO [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 47b13a1c78111d05c87ac4dbad84cbd4/A in TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4. 2024-11-22T03:41:30,264 INFO [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/A/e3be43fa335c470b8184a779503e3b29, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/A/b1584c8b0a4c49c2bfc79e77ad403d02, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/A/6d72541868314cb9ac09d5f073f3615f] into tmpdir=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/.tmp, totalSize=90.8 K 2024-11-22T03:41:30,264 INFO [RS:0;c85114ed5096:42545-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=53.85 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4. 2024-11-22T03:41:30,264 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4. files: [hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/A/e3be43fa335c470b8184a779503e3b29, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/A/b1584c8b0a4c49c2bfc79e77ad403d02, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/A/6d72541868314cb9ac09d5f073f3615f] 2024-11-22T03:41:30,264 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] compactions.Compactor(225): Compacting 7a56e285695344c1aad453081b77e8e7, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1732246885980 2024-11-22T03:41:30,265 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] compactions.Compactor(225): Compacting e3be43fa335c470b8184a779503e3b29, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1732246885980 2024-11-22T03:41:30,265 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] compactions.Compactor(225): Compacting b1584c8b0a4c49c2bfc79e77ad403d02, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=79, earliestPutTs=1732246887232 2024-11-22T03:41:30,265 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] compactions.Compactor(225): Compacting de8f517c62e6458cb317bdbea9af310a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=79, earliestPutTs=1732246887232 2024-11-22T03:41:30,266 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6d72541868314cb9ac09d5f073f3615f, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1732246889451 2024-11-22T03:41:30,266 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] compactions.Compactor(225): Compacting 642df1c01ae243c6b2994695f3c7460d, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1732246889451 2024-11-22T03:41:30,281 INFO [RS:0;c85114ed5096:42545-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 47b13a1c78111d05c87ac4dbad84cbd4#B#compaction#273 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-22T03:41:30,282 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/.tmp/B/a669db6d679e41978776b81d8f176460 is 50, key is test_row_0/B:col10/1732246889626/Put/seqid=0 2024-11-22T03:41:30,290 INFO [RS:0;c85114ed5096:42545-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=47b13a1c78111d05c87ac4dbad84cbd4] 2024-11-22T03:41:30,314 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241122cc10284a6fd04aa1bfcf5f07eded00f3_47b13a1c78111d05c87ac4dbad84cbd4 store=[table=TestAcidGuarantees family=A region=47b13a1c78111d05c87ac4dbad84cbd4] 2024-11-22T03:41:30,315 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241122cc10284a6fd04aa1bfcf5f07eded00f3_47b13a1c78111d05c87ac4dbad84cbd4, store=[table=TestAcidGuarantees family=A region=47b13a1c78111d05c87ac4dbad84cbd4] 2024-11-22T03:41:30,316 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241122cc10284a6fd04aa1bfcf5f07eded00f3_47b13a1c78111d05c87ac4dbad84cbd4 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=47b13a1c78111d05c87ac4dbad84cbd4] 2024-11-22T03:41:30,322 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742142_1318 (size=12207) 2024-11-22T03:41:30,336 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742143_1319 (size=4469) 2024-11-22T03:41:30,394 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42545 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-22T03:41:30,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4. 2024-11-22T03:41:30,395 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2902): Flushing 47b13a1c78111d05c87ac4dbad84cbd4 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-22T03:41:30,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-22T03:41:30,413 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241122b9a0c9e9dea64747b2f98f87508db16b_47b13a1c78111d05c87ac4dbad84cbd4 is 50, key is test_row_0/A:col10/1732246889774/Put/seqid=0 2024-11-22T03:41:30,433 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4. as already flushing 2024-11-22T03:41:30,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(8855): Flush requested on 47b13a1c78111d05c87ac4dbad84cbd4 2024-11-22T03:41:30,451 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742144_1320 (size=12154) 2024-11-22T03:41:30,467 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:30,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52458 deadline: 1732246950463, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:30,468 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:30,468 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:30,468 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:41:30,474 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:30,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52478 deadline: 1732246950466, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:30,476 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:30,477 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:30,477 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:41:30,478 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:30,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52486 deadline: 1732246950467, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:30,478 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:30,479 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:30,479 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:41:30,584 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:30,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52458 deadline: 1732246950579, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:30,585 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:30,585 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:30,585 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:41:30,594 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:30,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52478 deadline: 1732246950588, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:30,594 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:30,594 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:30,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52486 deadline: 1732246950588, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:30,594 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:30,594 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:41:30,596 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:30,596 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:30,596 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:41:30,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-22T03:41:30,733 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/.tmp/B/a669db6d679e41978776b81d8f176460 as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/B/a669db6d679e41978776b81d8f176460 2024-11-22T03:41:30,739 INFO [RS:0;c85114ed5096:42545-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 47b13a1c78111d05c87ac4dbad84cbd4#A#compaction#274 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-22T03:41:30,740 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/.tmp/A/900cb9e957a0467b843ffdb809c5b0bd is 175, key is test_row_0/A:col10/1732246889626/Put/seqid=0 2024-11-22T03:41:30,741 INFO [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 47b13a1c78111d05c87ac4dbad84cbd4/B of 47b13a1c78111d05c87ac4dbad84cbd4 into a669db6d679e41978776b81d8f176460(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T03:41:30,741 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 47b13a1c78111d05c87ac4dbad84cbd4: 2024-11-22T03:41:30,741 INFO [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., storeName=47b13a1c78111d05c87ac4dbad84cbd4/B, priority=13, startTime=1732246890261; duration=0sec 2024-11-22T03:41:30,741 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T03:41:30,741 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 47b13a1c78111d05c87ac4dbad84cbd4:B 2024-11-22T03:41:30,741 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T03:41:30,745 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33762 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T03:41:30,745 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HStore(1541): 47b13a1c78111d05c87ac4dbad84cbd4/C is initiating minor compaction (all files) 2024-11-22T03:41:30,745 INFO [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 47b13a1c78111d05c87ac4dbad84cbd4/C in TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4. 2024-11-22T03:41:30,745 INFO [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/C/fdaf9f11c8744da0abfd74f12f387052, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/C/80ea1b6c83f94b2c96effac7620874c6, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/C/5adf8b6acb714e57a81bbe590eb92096] into tmpdir=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/.tmp, totalSize=33.0 K 2024-11-22T03:41:30,745 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] compactions.Compactor(225): Compacting fdaf9f11c8744da0abfd74f12f387052, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1732246885980 2024-11-22T03:41:30,746 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] compactions.Compactor(225): Compacting 80ea1b6c83f94b2c96effac7620874c6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=79, earliestPutTs=1732246887232 2024-11-22T03:41:30,746 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] compactions.Compactor(225): Compacting 5adf8b6acb714e57a81bbe590eb92096, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1732246889451 2024-11-22T03:41:30,759 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742145_1321 (size=31268) 2024-11-22T03:41:30,762 INFO [RS:0;c85114ed5096:42545-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 47b13a1c78111d05c87ac4dbad84cbd4#C#compaction#276 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-22T03:41:30,762 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/.tmp/C/eb0db5ccd77d4cf3bfbaf64e3669fc33 is 50, key is test_row_0/C:col10/1732246889626/Put/seqid=0 2024-11-22T03:41:30,796 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:30,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52458 deadline: 1732246950789, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:30,797 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:30,797 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:30,797 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:41:30,816 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:30,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52486 deadline: 1732246950799, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:30,818 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:30,818 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:30,818 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:41:30,820 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:30,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52478 deadline: 1732246950799, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:30,823 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:30,823 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:30,823 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:41:30,827 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742146_1322 (size=12207) 2024-11-22T03:41:30,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:30,857 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241122b9a0c9e9dea64747b2f98f87508db16b_47b13a1c78111d05c87ac4dbad84cbd4 to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241122b9a0c9e9dea64747b2f98f87508db16b_47b13a1c78111d05c87ac4dbad84cbd4 2024-11-22T03:41:30,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/.tmp/A/a97433bf816d40b1940a4ffb7706f0ed, store: [table=TestAcidGuarantees family=A region=47b13a1c78111d05c87ac4dbad84cbd4] 2024-11-22T03:41:30,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/.tmp/A/a97433bf816d40b1940a4ffb7706f0ed is 175, key is test_row_0/A:col10/1732246889774/Put/seqid=0 2024-11-22T03:41:30,868 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742147_1323 (size=30955) 2024-11-22T03:41:30,870 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=117, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/.tmp/A/a97433bf816d40b1940a4ffb7706f0ed 2024-11-22T03:41:30,883 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/.tmp/B/d6907af7df39429da2da42cf852ed56b is 50, key is test_row_0/B:col10/1732246889774/Put/seqid=0 2024-11-22T03:41:30,906 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742148_1324 (size=12001) 2024-11-22T03:41:31,111 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:31,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52458 deadline: 1732246951108, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:31,111 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:31,111 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:31,111 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:41:31,133 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:31,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52486 deadline: 1732246951129, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:31,134 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:31,134 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:31,134 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:31,134 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:41:31,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52478 deadline: 1732246951129, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:31,134 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:31,134 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:31,134 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:41:31,167 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/.tmp/A/900cb9e957a0467b843ffdb809c5b0bd as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/A/900cb9e957a0467b843ffdb809c5b0bd 2024-11-22T03:41:31,173 INFO [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 47b13a1c78111d05c87ac4dbad84cbd4/A of 47b13a1c78111d05c87ac4dbad84cbd4 into 900cb9e957a0467b843ffdb809c5b0bd(size=30.5 K), total size for store is 30.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T03:41:31,173 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 47b13a1c78111d05c87ac4dbad84cbd4: 2024-11-22T03:41:31,173 INFO [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., storeName=47b13a1c78111d05c87ac4dbad84cbd4/A, priority=13, startTime=1732246890261; duration=0sec 2024-11-22T03:41:31,173 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T03:41:31,173 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 47b13a1c78111d05c87ac4dbad84cbd4:A 2024-11-22T03:41:31,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-22T03:41:31,246 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/.tmp/C/eb0db5ccd77d4cf3bfbaf64e3669fc33 as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/C/eb0db5ccd77d4cf3bfbaf64e3669fc33 2024-11-22T03:41:31,252 INFO [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 47b13a1c78111d05c87ac4dbad84cbd4/C of 47b13a1c78111d05c87ac4dbad84cbd4 into eb0db5ccd77d4cf3bfbaf64e3669fc33(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T03:41:31,252 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 47b13a1c78111d05c87ac4dbad84cbd4: 2024-11-22T03:41:31,252 INFO [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., storeName=47b13a1c78111d05c87ac4dbad84cbd4/C, priority=13, startTime=1732246890261; duration=0sec 2024-11-22T03:41:31,252 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T03:41:31,252 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 47b13a1c78111d05c87ac4dbad84cbd4:C 2024-11-22T03:41:31,307 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=117 (bloomFilter=true), to=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/.tmp/B/d6907af7df39429da2da42cf852ed56b 2024-11-22T03:41:31,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/.tmp/C/36c79ec139df46868e02775e2147d904 is 50, key is test_row_0/C:col10/1732246889774/Put/seqid=0 2024-11-22T03:41:31,372 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742149_1325 (size=12001) 2024-11-22T03:41:31,375 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=117 (bloomFilter=true), to=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/.tmp/C/36c79ec139df46868e02775e2147d904 2024-11-22T03:41:31,382 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/.tmp/A/a97433bf816d40b1940a4ffb7706f0ed as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/A/a97433bf816d40b1940a4ffb7706f0ed 2024-11-22T03:41:31,387 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/A/a97433bf816d40b1940a4ffb7706f0ed, entries=150, sequenceid=117, filesize=30.2 K 2024-11-22T03:41:31,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/.tmp/B/d6907af7df39429da2da42cf852ed56b as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/B/d6907af7df39429da2da42cf852ed56b 2024-11-22T03:41:31,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,393 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/B/d6907af7df39429da2da42cf852ed56b, entries=150, sequenceid=117, filesize=11.7 K 2024-11-22T03:41:31,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/.tmp/C/36c79ec139df46868e02775e2147d904 as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/C/36c79ec139df46868e02775e2147d904 2024-11-22T03:41:31,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,398 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/C/36c79ec139df46868e02775e2147d904, entries=150, sequenceid=117, filesize=11.7 K 2024-11-22T03:41:31,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,399 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 47b13a1c78111d05c87ac4dbad84cbd4 in 1005ms, sequenceid=117, compaction requested=false 2024-11-22T03:41:31,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,399 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2603): Flush status journal for 47b13a1c78111d05c87ac4dbad84cbd4: 2024-11-22T03:41:31,399 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4. 2024-11-22T03:41:31,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,399 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=98 2024-11-22T03:41:31,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42627 {}] master.HMaster(4169): Remote procedure done, pid=98 2024-11-22T03:41:31,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,403 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=98, resume processing ppid=97 2024-11-22T03:41:31,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,403 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=98, ppid=97, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3280 sec 2024-11-22T03:41:31,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,405 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=97, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees in 1.3340 sec 2024-11-22T03:41:31,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,643 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-22T03:41:31,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,643 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 47b13a1c78111d05c87ac4dbad84cbd4 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-22T03:41:31,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(8855): Flush requested on 47b13a1c78111d05c87ac4dbad84cbd4 2024-11-22T03:41:31,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,661 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112211d7ca2c98984f5e86e35a34f452207e_47b13a1c78111d05c87ac4dbad84cbd4 is 50, key is test_row_0/A:col10/1732246890462/Put/seqid=0 2024-11-22T03:41:31,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:31,673 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742150_1326 (size=12204) 2024-11-22T03:41:31,747 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:31,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52458 deadline: 1732246951735, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:31,747 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:31,747 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:31,747 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:31,747 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:41:31,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52478 deadline: 1732246951739, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:31,749 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:31,749 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:31,749 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:41:31,755 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:31,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52486 deadline: 1732246951743, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:31,756 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:31,756 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:31,756 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:41:31,856 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:31,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52458 deadline: 1732246951849, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:31,857 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:31,857 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:31,857 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:41:31,872 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:31,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52478 deadline: 1732246951859, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:31,873 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:31,872 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:31,873 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:31,873 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:41:31,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52486 deadline: 1732246951859, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:31,873 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:31,873 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:31,873 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:41:32,026 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:32,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52472 deadline: 1732246952018, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:32,027 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to c85114ed5096,42545,1732246786482 for 'test_row_2' in 47b13a1c78111d05c87ac4dbad84cbd4 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8219 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-22T03:41:32,028 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:32,028 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:32,028 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:41:32,048 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:32,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52414 deadline: 1732246952039, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:32,048 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to c85114ed5096,42545,1732246786482 for 'test_row_0' in 47b13a1c78111d05c87ac4dbad84cbd4 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8241 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-22T03:41:32,049 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:32,049 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:32,049 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:41:32,074 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:32,077 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:32,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52458 deadline: 1732246952074, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:32,079 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:32,079 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:32,079 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:41:32,082 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112211d7ca2c98984f5e86e35a34f452207e_47b13a1c78111d05c87ac4dbad84cbd4 to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112211d7ca2c98984f5e86e35a34f452207e_47b13a1c78111d05c87ac4dbad84cbd4 2024-11-22T03:41:32,083 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/.tmp/A/424bc496d28043e1a8b8cdb9b0c56b28, store: [table=TestAcidGuarantees family=A region=47b13a1c78111d05c87ac4dbad84cbd4] 2024-11-22T03:41:32,083 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/.tmp/A/424bc496d28043e1a8b8cdb9b0c56b28 is 175, key is test_row_0/A:col10/1732246890462/Put/seqid=0 2024-11-22T03:41:32,090 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:32,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52478 deadline: 1732246952079, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:32,090 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:32,090 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:32,090 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:41:32,090 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:32,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52486 deadline: 1732246952079, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:32,091 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:32,091 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:32,091 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:41:32,105 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742151_1327 (size=31005) 2024-11-22T03:41:32,106 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=132, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/.tmp/A/424bc496d28043e1a8b8cdb9b0c56b28 2024-11-22T03:41:32,118 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/.tmp/B/9582e788caca4cc0ae6d9576a3abb322 is 50, key is test_row_0/B:col10/1732246890462/Put/seqid=0 2024-11-22T03:41:32,127 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742152_1328 (size=12051) 2024-11-22T03:41:32,128 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=132 (bloomFilter=true), to=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/.tmp/B/9582e788caca4cc0ae6d9576a3abb322 2024-11-22T03:41:32,146 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/.tmp/C/f92470a0a7a748e98daa0cef10a47e3d is 50, key is test_row_0/C:col10/1732246890462/Put/seqid=0 2024-11-22T03:41:32,161 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742153_1329 (size=12051) 2024-11-22T03:41:32,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-22T03:41:32,229 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-22T03:41:32,229 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42627 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-22T03:41:32,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42627 {}] procedure2.ProcedureExecutor(1139): Stored pid=99, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees 2024-11-22T03:41:32,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-22T03:41:32,231 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=99, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-22T03:41:32,232 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=99, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-22T03:41:32,232 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=100, ppid=99, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-22T03:41:32,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-22T03:41:32,385 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42545 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-22T03:41:32,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4. 2024-11-22T03:41:32,386 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4. as already flushing 2024-11-22T03:41:32,386 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4. 2024-11-22T03:41:32,386 ERROR [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 47b13a1c78111d05c87ac4dbad84cbd4, NAME => 'TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:41:32,386 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 47b13a1c78111d05c87ac4dbad84cbd4, NAME => 'TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:41:32,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42627 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 47b13a1c78111d05c87ac4dbad84cbd4, NAME => 'TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 47b13a1c78111d05c87ac4dbad84cbd4, NAME => 'TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:41:32,396 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:32,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52458 deadline: 1732246952388, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:32,400 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:32,400 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:32,400 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:41:32,405 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:32,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52486 deadline: 1732246952399, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:32,406 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:32,406 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:32,406 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:32,406 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:41:32,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52478 deadline: 1732246952399, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:32,407 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:32,407 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:32,407 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:41:32,539 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42545 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-22T03:41:32,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4. 2024-11-22T03:41:32,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4. as already flushing 2024-11-22T03:41:32,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4. 2024-11-22T03:41:32,540 ERROR [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 47b13a1c78111d05c87ac4dbad84cbd4, NAME => 'TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:41:32,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 47b13a1c78111d05c87ac4dbad84cbd4, NAME => 'TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:41:32,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42627 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 47b13a1c78111d05c87ac4dbad84cbd4, NAME => 'TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 47b13a1c78111d05c87ac4dbad84cbd4, NAME => 'TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:41:32,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-22T03:41:32,560 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=132 (bloomFilter=true), to=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/.tmp/C/f92470a0a7a748e98daa0cef10a47e3d 2024-11-22T03:41:32,568 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/.tmp/A/424bc496d28043e1a8b8cdb9b0c56b28 as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/A/424bc496d28043e1a8b8cdb9b0c56b28 2024-11-22T03:41:32,575 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/A/424bc496d28043e1a8b8cdb9b0c56b28, entries=150, sequenceid=132, filesize=30.3 K 2024-11-22T03:41:32,575 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/.tmp/B/9582e788caca4cc0ae6d9576a3abb322 as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/B/9582e788caca4cc0ae6d9576a3abb322 2024-11-22T03:41:32,582 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/B/9582e788caca4cc0ae6d9576a3abb322, entries=150, sequenceid=132, filesize=11.8 K 2024-11-22T03:41:32,583 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/.tmp/C/f92470a0a7a748e98daa0cef10a47e3d as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/C/f92470a0a7a748e98daa0cef10a47e3d 2024-11-22T03:41:32,593 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/C/f92470a0a7a748e98daa0cef10a47e3d, entries=150, sequenceid=132, filesize=11.8 K 2024-11-22T03:41:32,594 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 47b13a1c78111d05c87ac4dbad84cbd4 in 950ms, sequenceid=132, compaction requested=true 2024-11-22T03:41:32,594 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 47b13a1c78111d05c87ac4dbad84cbd4: 2024-11-22T03:41:32,594 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T03:41:32,595 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 47b13a1c78111d05c87ac4dbad84cbd4:A, priority=-2147483648, current under compaction store size is 1 2024-11-22T03:41:32,595 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 93228 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T03:41:32,595 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.HStore(1541): 47b13a1c78111d05c87ac4dbad84cbd4/A is initiating minor compaction (all files) 2024-11-22T03:41:32,595 INFO [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 47b13a1c78111d05c87ac4dbad84cbd4/A in TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4. 2024-11-22T03:41:32,595 INFO [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/A/900cb9e957a0467b843ffdb809c5b0bd, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/A/a97433bf816d40b1940a4ffb7706f0ed, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/A/424bc496d28043e1a8b8cdb9b0c56b28] into tmpdir=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/.tmp, totalSize=91.0 K 2024-11-22T03:41:32,595 INFO [RS:0;c85114ed5096:42545-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=53.85 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4. 2024-11-22T03:41:32,595 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4. files: [hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/A/900cb9e957a0467b843ffdb809c5b0bd, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/A/a97433bf816d40b1940a4ffb7706f0ed, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/A/424bc496d28043e1a8b8cdb9b0c56b28] 2024-11-22T03:41:32,595 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] compactions.Compactor(225): Compacting 900cb9e957a0467b843ffdb809c5b0bd, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1732246887238 2024-11-22T03:41:32,595 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T03:41:32,596 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T03:41:32,596 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] compactions.Compactor(225): Compacting a97433bf816d40b1940a4ffb7706f0ed, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1732246889747 2024-11-22T03:41:32,596 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] compactions.Compactor(225): Compacting 424bc496d28043e1a8b8cdb9b0c56b28, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1732246890462 2024-11-22T03:41:32,597 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36259 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T03:41:32,597 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HStore(1541): 47b13a1c78111d05c87ac4dbad84cbd4/B is initiating minor compaction (all files) 2024-11-22T03:41:32,597 INFO [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 47b13a1c78111d05c87ac4dbad84cbd4/B in TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4. 2024-11-22T03:41:32,597 INFO [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/B/a669db6d679e41978776b81d8f176460, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/B/d6907af7df39429da2da42cf852ed56b, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/B/9582e788caca4cc0ae6d9576a3abb322] into tmpdir=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/.tmp, totalSize=35.4 K 2024-11-22T03:41:32,598 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] compactions.Compactor(225): Compacting a669db6d679e41978776b81d8f176460, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1732246887238 2024-11-22T03:41:32,598 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] compactions.Compactor(225): Compacting d6907af7df39429da2da42cf852ed56b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1732246889747 2024-11-22T03:41:32,598 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] compactions.Compactor(225): Compacting 9582e788caca4cc0ae6d9576a3abb322, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1732246890462 2024-11-22T03:41:32,600 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 47b13a1c78111d05c87ac4dbad84cbd4:B, priority=-2147483648, current under compaction store size is 2 2024-11-22T03:41:32,601 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T03:41:32,601 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 47b13a1c78111d05c87ac4dbad84cbd4:C, priority=-2147483648, current under compaction store size is 3 2024-11-22T03:41:32,601 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T03:41:32,618 INFO [RS:0;c85114ed5096:42545-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=47b13a1c78111d05c87ac4dbad84cbd4] 2024-11-22T03:41:32,638 INFO [RS:0;c85114ed5096:42545-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 47b13a1c78111d05c87ac4dbad84cbd4#B#compaction#283 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 53.85 MB/second 2024-11-22T03:41:32,639 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/.tmp/B/259e5ab7e8cd4b9cbe981d4a044e47c3 is 50, key is test_row_0/B:col10/1732246890462/Put/seqid=0 2024-11-22T03:41:32,650 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241122deabf7e9621f4f3398c589e5bf655a1f_47b13a1c78111d05c87ac4dbad84cbd4 store=[table=TestAcidGuarantees family=A region=47b13a1c78111d05c87ac4dbad84cbd4] 2024-11-22T03:41:32,652 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241122deabf7e9621f4f3398c589e5bf655a1f_47b13a1c78111d05c87ac4dbad84cbd4, store=[table=TestAcidGuarantees family=A region=47b13a1c78111d05c87ac4dbad84cbd4] 2024-11-22T03:41:32,652 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241122deabf7e9621f4f3398c589e5bf655a1f_47b13a1c78111d05c87ac4dbad84cbd4 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=47b13a1c78111d05c87ac4dbad84cbd4] 2024-11-22T03:41:32,668 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742155_1331 (size=4469) 2024-11-22T03:41:32,684 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742154_1330 (size=12359) 2024-11-22T03:41:32,692 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42545 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-22T03:41:32,692 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4. 2024-11-22T03:41:32,692 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2902): Flushing 47b13a1c78111d05c87ac4dbad84cbd4 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-22T03:41:32,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241122c91c444343b245159aba4269f75696f2_47b13a1c78111d05c87ac4dbad84cbd4 is 50, key is test_row_0/A:col10/1732246891741/Put/seqid=0 2024-11-22T03:41:32,742 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742156_1332 (size=12304) 2024-11-22T03:41:32,748 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:32,753 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241122c91c444343b245159aba4269f75696f2_47b13a1c78111d05c87ac4dbad84cbd4 to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241122c91c444343b245159aba4269f75696f2_47b13a1c78111d05c87ac4dbad84cbd4 2024-11-22T03:41:32,757 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/.tmp/A/a1f31d0f8f1846d3b1b6ea15e35eeecb, store: [table=TestAcidGuarantees family=A region=47b13a1c78111d05c87ac4dbad84cbd4] 2024-11-22T03:41:32,757 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/.tmp/A/a1f31d0f8f1846d3b1b6ea15e35eeecb is 175, key is test_row_0/A:col10/1732246891741/Put/seqid=0 2024-11-22T03:41:32,780 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742157_1333 (size=31105) 2024-11-22T03:41:32,781 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=156, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/.tmp/A/a1f31d0f8f1846d3b1b6ea15e35eeecb 2024-11-22T03:41:32,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/.tmp/B/f6213d5deb4c44feb5266a86d13f6cd2 is 50, key is test_row_0/B:col10/1732246891741/Put/seqid=0 2024-11-22T03:41:32,800 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742158_1334 (size=12151) 2024-11-22T03:41:32,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-22T03:41:32,911 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4. as already flushing 2024-11-22T03:41:32,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(8855): Flush requested on 47b13a1c78111d05c87ac4dbad84cbd4 2024-11-22T03:41:32,967 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:32,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52486 deadline: 1732246952949, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:32,968 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:32,968 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:32,968 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:41:32,968 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:32,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52458 deadline: 1732246952951, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:32,969 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:32,969 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:32,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52478 deadline: 1732246952954, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:32,969 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:32,969 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:41:32,969 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:32,969 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:32,969 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:41:33,072 INFO [RS:0;c85114ed5096:42545-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 47b13a1c78111d05c87ac4dbad84cbd4#A#compaction#282 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-22T03:41:33,072 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/.tmp/A/20aea355efa74c579cd8979ab04d7d46 is 175, key is test_row_0/A:col10/1732246890462/Put/seqid=0 2024-11-22T03:41:33,088 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:33,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52486 deadline: 1732246953079, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:33,089 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:33,089 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:33,089 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:41:33,089 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:33,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52458 deadline: 1732246953079, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:33,090 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:33,090 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:33,090 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:41:33,090 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:33,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52478 deadline: 1732246953079, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:33,091 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:33,091 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:33,091 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:41:33,091 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/.tmp/B/259e5ab7e8cd4b9cbe981d4a044e47c3 as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/B/259e5ab7e8cd4b9cbe981d4a044e47c3 2024-11-22T03:41:33,098 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742159_1335 (size=31313) 2024-11-22T03:41:33,099 INFO [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 47b13a1c78111d05c87ac4dbad84cbd4/B of 47b13a1c78111d05c87ac4dbad84cbd4 into 259e5ab7e8cd4b9cbe981d4a044e47c3(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T03:41:33,099 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 47b13a1c78111d05c87ac4dbad84cbd4: 2024-11-22T03:41:33,099 INFO [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., storeName=47b13a1c78111d05c87ac4dbad84cbd4/B, priority=13, startTime=1732246892596; duration=0sec 2024-11-22T03:41:33,099 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T03:41:33,099 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 47b13a1c78111d05c87ac4dbad84cbd4:B 2024-11-22T03:41:33,099 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T03:41:33,101 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36259 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T03:41:33,101 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HStore(1541): 47b13a1c78111d05c87ac4dbad84cbd4/C is initiating minor compaction (all files) 2024-11-22T03:41:33,101 INFO [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 47b13a1c78111d05c87ac4dbad84cbd4/C in TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4. 2024-11-22T03:41:33,101 INFO [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/C/eb0db5ccd77d4cf3bfbaf64e3669fc33, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/C/36c79ec139df46868e02775e2147d904, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/C/f92470a0a7a748e98daa0cef10a47e3d] into tmpdir=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/.tmp, totalSize=35.4 K 2024-11-22T03:41:33,102 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] compactions.Compactor(225): Compacting eb0db5ccd77d4cf3bfbaf64e3669fc33, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1732246887238 2024-11-22T03:41:33,102 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] compactions.Compactor(225): Compacting 36c79ec139df46868e02775e2147d904, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1732246889747 2024-11-22T03:41:33,103 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] compactions.Compactor(225): Compacting f92470a0a7a748e98daa0cef10a47e3d, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1732246890462 2024-11-22T03:41:33,103 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/.tmp/A/20aea355efa74c579cd8979ab04d7d46 as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/A/20aea355efa74c579cd8979ab04d7d46 2024-11-22T03:41:33,109 INFO [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 47b13a1c78111d05c87ac4dbad84cbd4/A of 47b13a1c78111d05c87ac4dbad84cbd4 into 20aea355efa74c579cd8979ab04d7d46(size=30.6 K), total size for store is 30.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T03:41:33,110 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 47b13a1c78111d05c87ac4dbad84cbd4: 2024-11-22T03:41:33,110 INFO [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., storeName=47b13a1c78111d05c87ac4dbad84cbd4/A, priority=13, startTime=1732246892594; duration=0sec 2024-11-22T03:41:33,110 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T03:41:33,110 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 47b13a1c78111d05c87ac4dbad84cbd4:A 2024-11-22T03:41:33,123 INFO [RS:0;c85114ed5096:42545-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 47b13a1c78111d05c87ac4dbad84cbd4#C#compaction#286 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-22T03:41:33,123 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/.tmp/C/26525a0fc0fb490d902d39bc1955bf76 is 50, key is test_row_0/C:col10/1732246890462/Put/seqid=0 2024-11-22T03:41:33,159 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742160_1336 (size=12359) 2024-11-22T03:41:33,200 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=156 (bloomFilter=true), to=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/.tmp/B/f6213d5deb4c44feb5266a86d13f6cd2 2024-11-22T03:41:33,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/.tmp/C/0949cc5df99b47edb00ea3acaae2efd5 is 50, key is test_row_0/C:col10/1732246891741/Put/seqid=0 2024-11-22T03:41:33,223 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742161_1337 (size=12151) 2024-11-22T03:41:33,307 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:33,307 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:33,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52458 deadline: 1732246953299, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:33,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52478 deadline: 1732246953299, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:33,307 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:33,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52486 deadline: 1732246953299, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:33,307 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:33,307 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:33,307 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:41:33,309 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:33,309 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:33,309 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:41:33,309 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:33,309 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:33,309 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:41:33,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-22T03:41:33,564 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/.tmp/C/26525a0fc0fb490d902d39bc1955bf76 as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/C/26525a0fc0fb490d902d39bc1955bf76 2024-11-22T03:41:33,569 INFO [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 47b13a1c78111d05c87ac4dbad84cbd4/C of 47b13a1c78111d05c87ac4dbad84cbd4 into 26525a0fc0fb490d902d39bc1955bf76(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T03:41:33,569 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 47b13a1c78111d05c87ac4dbad84cbd4: 2024-11-22T03:41:33,569 INFO [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., storeName=47b13a1c78111d05c87ac4dbad84cbd4/C, priority=13, startTime=1732246892601; duration=0sec 2024-11-22T03:41:33,570 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T03:41:33,570 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 47b13a1c78111d05c87ac4dbad84cbd4:C 2024-11-22T03:41:33,624 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=156 (bloomFilter=true), to=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/.tmp/C/0949cc5df99b47edb00ea3acaae2efd5 2024-11-22T03:41:33,625 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:33,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52458 deadline: 1732246953619, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:33,625 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:33,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52478 deadline: 1732246953619, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:33,626 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:33,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52486 deadline: 1732246953619, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:33,626 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:33,626 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:33,626 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:41:33,626 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:33,626 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:33,626 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:41:33,627 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:33,627 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:33,627 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:41:33,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/.tmp/A/a1f31d0f8f1846d3b1b6ea15e35eeecb as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/A/a1f31d0f8f1846d3b1b6ea15e35eeecb 2024-11-22T03:41:33,639 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/A/a1f31d0f8f1846d3b1b6ea15e35eeecb, entries=150, sequenceid=156, filesize=30.4 K 2024-11-22T03:41:33,640 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/.tmp/B/f6213d5deb4c44feb5266a86d13f6cd2 as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/B/f6213d5deb4c44feb5266a86d13f6cd2 2024-11-22T03:41:33,645 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/B/f6213d5deb4c44feb5266a86d13f6cd2, entries=150, sequenceid=156, filesize=11.9 K 2024-11-22T03:41:33,647 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/.tmp/C/0949cc5df99b47edb00ea3acaae2efd5 as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/C/0949cc5df99b47edb00ea3acaae2efd5 2024-11-22T03:41:33,651 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/C/0949cc5df99b47edb00ea3acaae2efd5, entries=150, sequenceid=156, filesize=11.9 K 2024-11-22T03:41:33,653 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for 47b13a1c78111d05c87ac4dbad84cbd4 in 961ms, sequenceid=156, compaction requested=false 2024-11-22T03:41:33,653 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2603): Flush status journal for 47b13a1c78111d05c87ac4dbad84cbd4: 2024-11-22T03:41:33,653 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4. 2024-11-22T03:41:33,653 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=100 2024-11-22T03:41:33,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42627 {}] master.HMaster(4169): Remote procedure done, pid=100 2024-11-22T03:41:33,660 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=100, resume processing ppid=99 2024-11-22T03:41:33,660 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=100, ppid=99, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4240 sec 2024-11-22T03:41:33,662 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=99, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees in 1.4320 sec 2024-11-22T03:41:34,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(8855): Flush requested on 47b13a1c78111d05c87ac4dbad84cbd4 2024-11-22T03:41:34,133 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-22T03:41:34,133 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 47b13a1c78111d05c87ac4dbad84cbd4 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-22T03:41:34,142 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241122f90652dad6cb43398e76618c1f02763e_47b13a1c78111d05c87ac4dbad84cbd4 is 50, key is test_row_0/A:col10/1732246892952/Put/seqid=0 2024-11-22T03:41:34,148 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742162_1338 (size=14794) 2024-11-22T03:41:34,191 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:34,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52486 deadline: 1732246954182, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:34,191 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:34,191 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:34,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52458 deadline: 1732246954184, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:34,191 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:34,191 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:41:34,191 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:34,191 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:34,191 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:34,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52478 deadline: 1732246954185, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:34,191 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:41:34,191 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:34,191 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:34,191 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:41:34,304 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:34,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52486 deadline: 1732246954299, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:34,305 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:34,305 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:34,305 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:34,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52458 deadline: 1732246954299, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:34,305 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:41:34,305 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:34,305 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:34,305 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:34,305 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:41:34,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52478 deadline: 1732246954299, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:34,305 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:34,305 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:34,305 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:41:34,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-22T03:41:34,379 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-22T03:41:34,379 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42627 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-22T03:41:34,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42627 {}] procedure2.ProcedureExecutor(1139): Stored pid=101, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees 2024-11-22T03:41:34,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-22T03:41:34,381 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=101, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-22T03:41:34,381 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=101, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-22T03:41:34,381 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=102, ppid=101, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-22T03:41:34,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-22T03:41:34,514 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:34,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52486 deadline: 1732246954509, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:34,514 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:34,514 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:34,514 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:34,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52458 deadline: 1732246954509, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:34,514 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:41:34,514 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:34,514 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:34,514 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:41:34,514 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:34,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52478 deadline: 1732246954509, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:34,515 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:34,515 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:34,515 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:41:34,533 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42545 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-22T03:41:34,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4. 2024-11-22T03:41:34,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4. as already flushing 2024-11-22T03:41:34,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4. 2024-11-22T03:41:34,534 ERROR [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 47b13a1c78111d05c87ac4dbad84cbd4, NAME => 'TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:41:34,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 47b13a1c78111d05c87ac4dbad84cbd4, NAME => 'TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:41:34,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42627 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 47b13a1c78111d05c87ac4dbad84cbd4, NAME => 'TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 47b13a1c78111d05c87ac4dbad84cbd4, NAME => 'TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:41:34,553 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:34,556 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241122f90652dad6cb43398e76618c1f02763e_47b13a1c78111d05c87ac4dbad84cbd4 to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241122f90652dad6cb43398e76618c1f02763e_47b13a1c78111d05c87ac4dbad84cbd4 2024-11-22T03:41:34,557 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/.tmp/A/511097a8881648ee82f0523a63983cfa, store: [table=TestAcidGuarantees family=A region=47b13a1c78111d05c87ac4dbad84cbd4] 2024-11-22T03:41:34,557 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/.tmp/A/511097a8881648ee82f0523a63983cfa is 175, key is test_row_0/A:col10/1732246892952/Put/seqid=0 2024-11-22T03:41:34,565 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742163_1339 (size=39749) 2024-11-22T03:41:34,686 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42545 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-22T03:41:34,686 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4. 2024-11-22T03:41:34,686 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4. as already flushing 2024-11-22T03:41:34,686 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4. 2024-11-22T03:41:34,686 ERROR [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 47b13a1c78111d05c87ac4dbad84cbd4, NAME => 'TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:41:34,686 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 47b13a1c78111d05c87ac4dbad84cbd4, NAME => 'TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:41:34,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42627 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 47b13a1c78111d05c87ac4dbad84cbd4, NAME => 'TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 47b13a1c78111d05c87ac4dbad84cbd4, NAME => 'TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:41:34,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-22T03:41:34,824 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:34,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52486 deadline: 1732246954819, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:34,825 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:34,825 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:34,825 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:34,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52458 deadline: 1732246954819, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:34,825 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:41:34,825 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:34,825 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:34,825 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:34,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52478 deadline: 1732246954819, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:34,825 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:41:34,825 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:34,825 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:34,825 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:41:34,838 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42545 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-22T03:41:34,839 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4. 2024-11-22T03:41:34,839 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4. as already flushing 2024-11-22T03:41:34,839 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4. 2024-11-22T03:41:34,839 ERROR [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 47b13a1c78111d05c87ac4dbad84cbd4, NAME => 'TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:41:34,839 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 47b13a1c78111d05c87ac4dbad84cbd4, NAME => 'TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:41:34,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42627 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 47b13a1c78111d05c87ac4dbad84cbd4, NAME => 'TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 47b13a1c78111d05c87ac4dbad84cbd4, NAME => 'TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:41:34,962 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=172, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/.tmp/A/511097a8881648ee82f0523a63983cfa 2024-11-22T03:41:34,969 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/.tmp/B/4f834fc3ea1b40389e3f6368f244c2dc is 50, key is test_row_0/B:col10/1732246892952/Put/seqid=0 2024-11-22T03:41:34,990 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742164_1340 (size=12151) 2024-11-22T03:41:34,991 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=172 (bloomFilter=true), to=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/.tmp/B/4f834fc3ea1b40389e3f6368f244c2dc 2024-11-22T03:41:34,997 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42545 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-22T03:41:34,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4. 2024-11-22T03:41:34,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4. as already flushing 2024-11-22T03:41:34,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4. 2024-11-22T03:41:34,997 ERROR [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 47b13a1c78111d05c87ac4dbad84cbd4, NAME => 'TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:41:34,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 47b13a1c78111d05c87ac4dbad84cbd4, NAME => 'TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:41:34,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42627 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 47b13a1c78111d05c87ac4dbad84cbd4, NAME => 'TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 47b13a1c78111d05c87ac4dbad84cbd4, NAME => 'TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:41:35,006 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/.tmp/C/b3cc8352893e4c44bf65783ae90db563 is 50, key is test_row_0/C:col10/1732246892952/Put/seqid=0 2024-11-22T03:41:35,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-22T03:41:35,028 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742165_1341 (size=12151) 2024-11-22T03:41:35,150 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42545 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-22T03:41:35,151 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4. 2024-11-22T03:41:35,151 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4. as already flushing 2024-11-22T03:41:35,151 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4. 2024-11-22T03:41:35,151 ERROR [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 47b13a1c78111d05c87ac4dbad84cbd4, NAME => 'TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:41:35,151 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 47b13a1c78111d05c87ac4dbad84cbd4, NAME => 'TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:41:35,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42627 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 47b13a1c78111d05c87ac4dbad84cbd4, NAME => 'TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 47b13a1c78111d05c87ac4dbad84cbd4, NAME => 'TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:41:35,304 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42545 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-22T03:41:35,304 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4. 2024-11-22T03:41:35,304 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4. as already flushing 2024-11-22T03:41:35,304 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4. 2024-11-22T03:41:35,304 ERROR [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 47b13a1c78111d05c87ac4dbad84cbd4, NAME => 'TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:41:35,304 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 47b13a1c78111d05c87ac4dbad84cbd4, NAME => 'TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:41:35,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42627 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 47b13a1c78111d05c87ac4dbad84cbd4, NAME => 'TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 47b13a1c78111d05c87ac4dbad84cbd4, NAME => 'TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:41:35,336 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:35,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52486 deadline: 1732246955328, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:35,337 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:35,337 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:35,337 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:41:35,343 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:35,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52458 deadline: 1732246955339, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:35,344 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:35,344 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:35,344 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:41:35,344 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:35,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52478 deadline: 1732246955339, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:35,345 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:35,345 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:35,345 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:41:35,428 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=172 (bloomFilter=true), to=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/.tmp/C/b3cc8352893e4c44bf65783ae90db563 2024-11-22T03:41:35,435 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/.tmp/A/511097a8881648ee82f0523a63983cfa as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/A/511097a8881648ee82f0523a63983cfa 2024-11-22T03:41:35,438 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/A/511097a8881648ee82f0523a63983cfa, entries=200, sequenceid=172, filesize=38.8 K 2024-11-22T03:41:35,439 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/.tmp/B/4f834fc3ea1b40389e3f6368f244c2dc as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/B/4f834fc3ea1b40389e3f6368f244c2dc 2024-11-22T03:41:35,442 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/B/4f834fc3ea1b40389e3f6368f244c2dc, entries=150, sequenceid=172, filesize=11.9 K 2024-11-22T03:41:35,443 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/.tmp/C/b3cc8352893e4c44bf65783ae90db563 as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/C/b3cc8352893e4c44bf65783ae90db563 2024-11-22T03:41:35,458 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42545 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-22T03:41:35,458 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4. 2024-11-22T03:41:35,458 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4. as already flushing 2024-11-22T03:41:35,458 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4. 2024-11-22T03:41:35,458 ERROR [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 47b13a1c78111d05c87ac4dbad84cbd4, NAME => 'TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:41:35,458 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 47b13a1c78111d05c87ac4dbad84cbd4, NAME => 'TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:41:35,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42627 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 47b13a1c78111d05c87ac4dbad84cbd4, NAME => 'TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 47b13a1c78111d05c87ac4dbad84cbd4, NAME => 'TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:41:35,463 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/C/b3cc8352893e4c44bf65783ae90db563, entries=150, sequenceid=172, filesize=11.9 K 2024-11-22T03:41:35,464 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for 47b13a1c78111d05c87ac4dbad84cbd4 in 1331ms, sequenceid=172, compaction requested=true 2024-11-22T03:41:35,464 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 47b13a1c78111d05c87ac4dbad84cbd4: 2024-11-22T03:41:35,464 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 47b13a1c78111d05c87ac4dbad84cbd4:A, priority=-2147483648, current under compaction store size is 1 2024-11-22T03:41:35,464 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T03:41:35,464 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T03:41:35,464 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 47b13a1c78111d05c87ac4dbad84cbd4:B, priority=-2147483648, current under compaction store size is 2 2024-11-22T03:41:35,464 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T03:41:35,464 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T03:41:35,464 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 47b13a1c78111d05c87ac4dbad84cbd4:C, priority=-2147483648, current under compaction store size is 3 2024-11-22T03:41:35,464 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T03:41:35,465 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102167 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T03:41:35,465 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.HStore(1541): 47b13a1c78111d05c87ac4dbad84cbd4/A is initiating minor compaction (all files) 2024-11-22T03:41:35,465 INFO [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 47b13a1c78111d05c87ac4dbad84cbd4/A in TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4. 2024-11-22T03:41:35,465 INFO [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/A/20aea355efa74c579cd8979ab04d7d46, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/A/a1f31d0f8f1846d3b1b6ea15e35eeecb, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/A/511097a8881648ee82f0523a63983cfa] into tmpdir=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/.tmp, totalSize=99.8 K 2024-11-22T03:41:35,465 INFO [RS:0;c85114ed5096:42545-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=53.85 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4. 2024-11-22T03:41:35,465 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4. files: [hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/A/20aea355efa74c579cd8979ab04d7d46, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/A/a1f31d0f8f1846d3b1b6ea15e35eeecb, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/A/511097a8881648ee82f0523a63983cfa] 2024-11-22T03:41:35,466 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36661 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T03:41:35,466 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HStore(1541): 47b13a1c78111d05c87ac4dbad84cbd4/B is initiating minor compaction (all files) 2024-11-22T03:41:35,466 INFO [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 47b13a1c78111d05c87ac4dbad84cbd4/B in TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4. 2024-11-22T03:41:35,466 INFO [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/B/259e5ab7e8cd4b9cbe981d4a044e47c3, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/B/f6213d5deb4c44feb5266a86d13f6cd2, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/B/4f834fc3ea1b40389e3f6368f244c2dc] into tmpdir=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/.tmp, totalSize=35.8 K 2024-11-22T03:41:35,466 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] compactions.Compactor(225): Compacting 20aea355efa74c579cd8979ab04d7d46, keycount=150, bloomtype=ROW, size=30.6 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1732246890462 2024-11-22T03:41:35,466 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] compactions.Compactor(225): Compacting 259e5ab7e8cd4b9cbe981d4a044e47c3, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1732246890462 2024-11-22T03:41:35,466 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] compactions.Compactor(225): Compacting a1f31d0f8f1846d3b1b6ea15e35eeecb, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1732246891731 2024-11-22T03:41:35,467 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] compactions.Compactor(225): Compacting f6213d5deb4c44feb5266a86d13f6cd2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1732246891731 2024-11-22T03:41:35,467 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] compactions.Compactor(225): Compacting 511097a8881648ee82f0523a63983cfa, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1732246892941 2024-11-22T03:41:35,467 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] compactions.Compactor(225): Compacting 4f834fc3ea1b40389e3f6368f244c2dc, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1732246892941 2024-11-22T03:41:35,485 INFO [RS:0;c85114ed5096:42545-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=47b13a1c78111d05c87ac4dbad84cbd4] 2024-11-22T03:41:35,503 INFO [RS:0;c85114ed5096:42545-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 47b13a1c78111d05c87ac4dbad84cbd4#B#compaction#292 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 53.85 MB/second 2024-11-22T03:41:35,503 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/.tmp/B/5f375b04bd204edb87f27a5841598d06 is 50, key is test_row_0/B:col10/1732246892952/Put/seqid=0 2024-11-22T03:41:35,514 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024112274d7618e8bda4b49b50fefabbe32f07e_47b13a1c78111d05c87ac4dbad84cbd4 store=[table=TestAcidGuarantees family=A region=47b13a1c78111d05c87ac4dbad84cbd4] 2024-11-22T03:41:35,516 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024112274d7618e8bda4b49b50fefabbe32f07e_47b13a1c78111d05c87ac4dbad84cbd4, store=[table=TestAcidGuarantees family=A region=47b13a1c78111d05c87ac4dbad84cbd4] 2024-11-22T03:41:35,516 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112274d7618e8bda4b49b50fefabbe32f07e_47b13a1c78111d05c87ac4dbad84cbd4 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=47b13a1c78111d05c87ac4dbad84cbd4] 2024-11-22T03:41:35,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-22T03:41:35,531 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742166_1342 (size=12561) 2024-11-22T03:41:35,540 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/.tmp/B/5f375b04bd204edb87f27a5841598d06 as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/B/5f375b04bd204edb87f27a5841598d06 2024-11-22T03:41:35,543 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742167_1343 (size=4469) 2024-11-22T03:41:35,545 INFO [RS:0;c85114ed5096:42545-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 47b13a1c78111d05c87ac4dbad84cbd4#A#compaction#291 average throughput is 0.41 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-22T03:41:35,546 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/.tmp/A/1420a2f7b6b841d7a8bf591779f57bba is 175, key is test_row_0/A:col10/1732246892952/Put/seqid=0 2024-11-22T03:41:35,546 INFO [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 47b13a1c78111d05c87ac4dbad84cbd4/B of 47b13a1c78111d05c87ac4dbad84cbd4 into 5f375b04bd204edb87f27a5841598d06(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T03:41:35,546 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 47b13a1c78111d05c87ac4dbad84cbd4: 2024-11-22T03:41:35,546 INFO [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., storeName=47b13a1c78111d05c87ac4dbad84cbd4/B, priority=13, startTime=1732246895464; duration=0sec 2024-11-22T03:41:35,546 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T03:41:35,546 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 47b13a1c78111d05c87ac4dbad84cbd4:B 2024-11-22T03:41:35,546 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T03:41:35,548 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36661 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T03:41:35,548 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HStore(1541): 47b13a1c78111d05c87ac4dbad84cbd4/C is initiating minor compaction (all files) 2024-11-22T03:41:35,548 INFO [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 47b13a1c78111d05c87ac4dbad84cbd4/C in TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4. 2024-11-22T03:41:35,549 INFO [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/C/26525a0fc0fb490d902d39bc1955bf76, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/C/0949cc5df99b47edb00ea3acaae2efd5, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/C/b3cc8352893e4c44bf65783ae90db563] into tmpdir=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/.tmp, totalSize=35.8 K 2024-11-22T03:41:35,549 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] compactions.Compactor(225): Compacting 26525a0fc0fb490d902d39bc1955bf76, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1732246890462 2024-11-22T03:41:35,549 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] compactions.Compactor(225): Compacting 0949cc5df99b47edb00ea3acaae2efd5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1732246891731 2024-11-22T03:41:35,549 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] compactions.Compactor(225): Compacting b3cc8352893e4c44bf65783ae90db563, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1732246892941 2024-11-22T03:41:35,552 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742168_1344 (size=31515) 2024-11-22T03:41:35,557 INFO [RS:0;c85114ed5096:42545-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 47b13a1c78111d05c87ac4dbad84cbd4#C#compaction#293 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-22T03:41:35,559 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/.tmp/C/e476a4c625644afc8f837f4f8ec07741 is 50, key is test_row_0/C:col10/1732246892952/Put/seqid=0 2024-11-22T03:41:35,565 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742169_1345 (size=12561) 2024-11-22T03:41:35,610 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42545 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-22T03:41:35,611 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4. 2024-11-22T03:41:35,611 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2902): Flushing 47b13a1c78111d05c87ac4dbad84cbd4 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-22T03:41:35,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241122f2d5c73d76934823abd42559eaf16da2_47b13a1c78111d05c87ac4dbad84cbd4 is 50, key is test_row_0/A:col10/1732246894180/Put/seqid=0 2024-11-22T03:41:35,625 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742170_1346 (size=12304) 2024-11-22T03:41:35,957 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/.tmp/A/1420a2f7b6b841d7a8bf591779f57bba as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/A/1420a2f7b6b841d7a8bf591779f57bba 2024-11-22T03:41:35,963 INFO [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 47b13a1c78111d05c87ac4dbad84cbd4/A of 47b13a1c78111d05c87ac4dbad84cbd4 into 1420a2f7b6b841d7a8bf591779f57bba(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T03:41:35,963 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 47b13a1c78111d05c87ac4dbad84cbd4: 2024-11-22T03:41:35,963 INFO [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., storeName=47b13a1c78111d05c87ac4dbad84cbd4/A, priority=13, startTime=1732246895464; duration=0sec 2024-11-22T03:41:35,963 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T03:41:35,963 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 47b13a1c78111d05c87ac4dbad84cbd4:A 2024-11-22T03:41:35,971 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/.tmp/C/e476a4c625644afc8f837f4f8ec07741 as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/C/e476a4c625644afc8f837f4f8ec07741 2024-11-22T03:41:35,978 INFO [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 47b13a1c78111d05c87ac4dbad84cbd4/C of 47b13a1c78111d05c87ac4dbad84cbd4 into e476a4c625644afc8f837f4f8ec07741(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T03:41:35,978 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 47b13a1c78111d05c87ac4dbad84cbd4: 2024-11-22T03:41:35,978 INFO [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., storeName=47b13a1c78111d05c87ac4dbad84cbd4/C, priority=13, startTime=1732246895464; duration=0sec 2024-11-22T03:41:35,978 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T03:41:35,978 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 47b13a1c78111d05c87ac4dbad84cbd4:C 2024-11-22T03:41:36,026 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:36,029 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241122f2d5c73d76934823abd42559eaf16da2_47b13a1c78111d05c87ac4dbad84cbd4 to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241122f2d5c73d76934823abd42559eaf16da2_47b13a1c78111d05c87ac4dbad84cbd4 2024-11-22T03:41:36,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/.tmp/A/73650cb54f9142bf8655a34da8b92b9a, store: [table=TestAcidGuarantees family=A region=47b13a1c78111d05c87ac4dbad84cbd4] 2024-11-22T03:41:36,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/.tmp/A/73650cb54f9142bf8655a34da8b92b9a is 175, key is test_row_0/A:col10/1732246894180/Put/seqid=0 2024-11-22T03:41:36,040 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742171_1347 (size=31105) 2024-11-22T03:41:36,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(8855): Flush requested on 47b13a1c78111d05c87ac4dbad84cbd4 2024-11-22T03:41:36,351 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4. as already flushing 2024-11-22T03:41:36,381 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:36,381 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:36,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52486 deadline: 1732246956375, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:36,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52478 deadline: 1732246956375, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:36,382 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:36,382 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:36,382 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:36,382 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:41:36,382 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:36,382 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:41:36,382 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:36,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52458 deadline: 1732246956376, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:36,383 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:36,383 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:36,383 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:41:36,440 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=196, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/.tmp/A/73650cb54f9142bf8655a34da8b92b9a 2024-11-22T03:41:36,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/.tmp/B/742d540140d04a42b34418be4489f47f is 50, key is test_row_0/B:col10/1732246894180/Put/seqid=0 2024-11-22T03:41:36,462 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742172_1348 (size=12151) 2024-11-22T03:41:36,466 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=196 (bloomFilter=true), to=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/.tmp/B/742d540140d04a42b34418be4489f47f 2024-11-22T03:41:36,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/.tmp/C/19e2207df0f44c29bb29a3fca70a6791 is 50, key is test_row_0/C:col10/1732246894180/Put/seqid=0 2024-11-22T03:41:36,479 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742173_1349 (size=12151) 2024-11-22T03:41:36,495 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:36,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52478 deadline: 1732246956489, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:36,495 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:36,495 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:36,495 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:36,495 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:41:36,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52458 deadline: 1732246956489, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:36,496 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:36,496 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:36,496 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:41:36,496 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:36,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52486 deadline: 1732246956489, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:36,496 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:36,496 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:36,496 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:41:36,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-22T03:41:36,702 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:36,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52478 deadline: 1732246956698, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:36,703 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:36,703 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:36,703 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:36,703 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:41:36,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52486 deadline: 1732246956698, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:36,703 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:36,703 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:36,703 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:41:36,714 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:36,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52458 deadline: 1732246956709, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:36,715 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:36,715 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:36,715 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:41:36,880 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=196 (bloomFilter=true), to=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/.tmp/C/19e2207df0f44c29bb29a3fca70a6791 2024-11-22T03:41:36,883 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/.tmp/A/73650cb54f9142bf8655a34da8b92b9a as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/A/73650cb54f9142bf8655a34da8b92b9a 2024-11-22T03:41:36,887 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/A/73650cb54f9142bf8655a34da8b92b9a, entries=150, sequenceid=196, filesize=30.4 K 2024-11-22T03:41:36,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/.tmp/B/742d540140d04a42b34418be4489f47f as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/B/742d540140d04a42b34418be4489f47f 2024-11-22T03:41:36,892 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/B/742d540140d04a42b34418be4489f47f, entries=150, sequenceid=196, filesize=11.9 K 2024-11-22T03:41:36,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/.tmp/C/19e2207df0f44c29bb29a3fca70a6791 as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/C/19e2207df0f44c29bb29a3fca70a6791 2024-11-22T03:41:36,895 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/C/19e2207df0f44c29bb29a3fca70a6791, entries=150, sequenceid=196, filesize=11.9 K 2024-11-22T03:41:36,896 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for 47b13a1c78111d05c87ac4dbad84cbd4 in 1285ms, sequenceid=196, compaction requested=false 2024-11-22T03:41:36,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2603): Flush status journal for 47b13a1c78111d05c87ac4dbad84cbd4: 2024-11-22T03:41:36,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4. 2024-11-22T03:41:36,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=102 2024-11-22T03:41:36,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42627 {}] master.HMaster(4169): Remote procedure done, pid=102 2024-11-22T03:41:36,898 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=102, resume processing ppid=101 2024-11-22T03:41:36,898 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=102, ppid=101, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.5160 sec 2024-11-22T03:41:36,900 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=101, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees in 2.5200 sec 2024-11-22T03:41:37,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(8855): Flush requested on 47b13a1c78111d05c87ac4dbad84cbd4 2024-11-22T03:41:37,012 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-22T03:41:37,012 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 47b13a1c78111d05c87ac4dbad84cbd4 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-22T03:41:37,022 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241122eb6f5239900345108db8d804dd4895a7_47b13a1c78111d05c87ac4dbad84cbd4 is 50, key is test_row_0/A:col10/1732246896374/Put/seqid=0 2024-11-22T03:41:37,034 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742174_1350 (size=14794) 2024-11-22T03:41:37,078 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:37,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52478 deadline: 1732246957068, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:37,079 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:37,079 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:37,079 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:41:37,090 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:37,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52486 deadline: 1732246957080, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:37,090 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:37,090 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:37,090 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:37,090 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:41:37,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52458 deadline: 1732246957081, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:37,091 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:37,091 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:37,091 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:41:37,198 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:37,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52478 deadline: 1732246957189, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:37,199 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:37,199 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:37,199 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:41:37,206 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:37,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52458 deadline: 1732246957198, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:37,206 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:37,206 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:37,207 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:37,207 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:41:37,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52486 deadline: 1732246957199, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:37,207 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:37,207 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:37,207 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:41:37,417 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:37,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52478 deadline: 1732246957408, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:37,417 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:37,417 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:37,417 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:41:37,417 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:37,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52458 deadline: 1732246957408, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:37,418 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:37,418 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:37,418 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:41:37,429 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:37,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52486 deadline: 1732246957418, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:37,430 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:37,430 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:37,430 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:41:37,434 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:37,437 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241122eb6f5239900345108db8d804dd4895a7_47b13a1c78111d05c87ac4dbad84cbd4 to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241122eb6f5239900345108db8d804dd4895a7_47b13a1c78111d05c87ac4dbad84cbd4 2024-11-22T03:41:37,438 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/.tmp/A/6fcc9c37ffcb4a0da2008f32d9bfc6c4, store: [table=TestAcidGuarantees family=A region=47b13a1c78111d05c87ac4dbad84cbd4] 2024-11-22T03:41:37,438 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/.tmp/A/6fcc9c37ffcb4a0da2008f32d9bfc6c4 is 175, key is test_row_0/A:col10/1732246896374/Put/seqid=0 2024-11-22T03:41:37,449 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742175_1351 (size=39749) 2024-11-22T03:41:37,736 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:37,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52478 deadline: 1732246957729, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:37,736 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:37,736 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:37,736 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:37,736 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:41:37,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52458 deadline: 1732246957729, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:37,737 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:37,737 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:37,737 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:41:37,746 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:37,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52486 deadline: 1732246957738, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:37,746 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:37,746 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:37,747 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:41:37,848 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=212, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/.tmp/A/6fcc9c37ffcb4a0da2008f32d9bfc6c4 2024-11-22T03:41:37,854 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/.tmp/B/1257b2ad4ad5486386a698507848789d is 50, key is test_row_0/B:col10/1732246896374/Put/seqid=0 2024-11-22T03:41:37,858 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742176_1352 (size=12151) 2024-11-22T03:41:37,858 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=212 (bloomFilter=true), to=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/.tmp/B/1257b2ad4ad5486386a698507848789d 2024-11-22T03:41:37,869 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/.tmp/C/516c3d5a186e4d159a071993f4ca8771 is 50, key is test_row_0/C:col10/1732246896374/Put/seqid=0 2024-11-22T03:41:37,878 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742177_1353 (size=12151) 2024-11-22T03:41:38,253 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:38,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52478 deadline: 1732246958248, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:38,254 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:38,253 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:38,254 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:38,254 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:41:38,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52458 deadline: 1732246958248, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:38,254 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:38,254 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:38,254 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:41:38,264 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:38,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52486 deadline: 1732246958258, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:38,265 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:38,265 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:38,265 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:41:38,282 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=212 (bloomFilter=true), to=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/.tmp/C/516c3d5a186e4d159a071993f4ca8771 2024-11-22T03:41:38,286 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/.tmp/A/6fcc9c37ffcb4a0da2008f32d9bfc6c4 as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/A/6fcc9c37ffcb4a0da2008f32d9bfc6c4 2024-11-22T03:41:38,289 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/A/6fcc9c37ffcb4a0da2008f32d9bfc6c4, entries=200, sequenceid=212, filesize=38.8 K 2024-11-22T03:41:38,289 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/.tmp/B/1257b2ad4ad5486386a698507848789d as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/B/1257b2ad4ad5486386a698507848789d 2024-11-22T03:41:38,292 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/B/1257b2ad4ad5486386a698507848789d, entries=150, sequenceid=212, filesize=11.9 K 2024-11-22T03:41:38,293 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/.tmp/C/516c3d5a186e4d159a071993f4ca8771 as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/C/516c3d5a186e4d159a071993f4ca8771 2024-11-22T03:41:38,296 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/C/516c3d5a186e4d159a071993f4ca8771, entries=150, sequenceid=212, filesize=11.9 K 2024-11-22T03:41:38,297 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=134.18 KB/137400 for 47b13a1c78111d05c87ac4dbad84cbd4 in 1285ms, sequenceid=212, compaction requested=true 2024-11-22T03:41:38,297 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 47b13a1c78111d05c87ac4dbad84cbd4: 2024-11-22T03:41:38,297 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 47b13a1c78111d05c87ac4dbad84cbd4:A, priority=-2147483648, current under compaction store size is 1 2024-11-22T03:41:38,297 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T03:41:38,297 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 47b13a1c78111d05c87ac4dbad84cbd4:B, priority=-2147483648, current under compaction store size is 2 2024-11-22T03:41:38,297 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T03:41:38,297 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T03:41:38,297 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 47b13a1c78111d05c87ac4dbad84cbd4:C, priority=-2147483648, current under compaction store size is 3 2024-11-22T03:41:38,297 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T03:41:38,297 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T03:41:38,298 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102369 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T03:41:38,298 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.HStore(1541): 47b13a1c78111d05c87ac4dbad84cbd4/A is initiating minor compaction (all files) 2024-11-22T03:41:38,298 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T03:41:38,298 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HStore(1541): 47b13a1c78111d05c87ac4dbad84cbd4/B is initiating minor compaction (all files) 2024-11-22T03:41:38,298 INFO [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 47b13a1c78111d05c87ac4dbad84cbd4/A in TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4. 2024-11-22T03:41:38,298 INFO [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 47b13a1c78111d05c87ac4dbad84cbd4/B in TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4. 2024-11-22T03:41:38,298 INFO [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/B/5f375b04bd204edb87f27a5841598d06, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/B/742d540140d04a42b34418be4489f47f, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/B/1257b2ad4ad5486386a698507848789d] into tmpdir=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/.tmp, totalSize=36.0 K 2024-11-22T03:41:38,298 INFO [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/A/1420a2f7b6b841d7a8bf591779f57bba, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/A/73650cb54f9142bf8655a34da8b92b9a, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/A/6fcc9c37ffcb4a0da2008f32d9bfc6c4] into tmpdir=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/.tmp, totalSize=100.0 K 2024-11-22T03:41:38,298 INFO [RS:0;c85114ed5096:42545-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=53.85 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4. 2024-11-22T03:41:38,298 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4. files: [hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/A/1420a2f7b6b841d7a8bf591779f57bba, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/A/73650cb54f9142bf8655a34da8b92b9a, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/A/6fcc9c37ffcb4a0da2008f32d9bfc6c4] 2024-11-22T03:41:38,298 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] compactions.Compactor(225): Compacting 5f375b04bd204edb87f27a5841598d06, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1732246892941 2024-11-22T03:41:38,299 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1420a2f7b6b841d7a8bf591779f57bba, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1732246892941 2024-11-22T03:41:38,305 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] compactions.Compactor(225): Compacting 742d540140d04a42b34418be4489f47f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=196, earliestPutTs=1732246894180 2024-11-22T03:41:38,305 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] compactions.Compactor(225): Compacting 73650cb54f9142bf8655a34da8b92b9a, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=196, earliestPutTs=1732246894180 2024-11-22T03:41:38,306 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] compactions.Compactor(225): Compacting 1257b2ad4ad5486386a698507848789d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=212, earliestPutTs=1732246896364 2024-11-22T03:41:38,306 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6fcc9c37ffcb4a0da2008f32d9bfc6c4, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=212, earliestPutTs=1732246896364 2024-11-22T03:41:38,316 INFO [RS:0;c85114ed5096:42545-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=47b13a1c78111d05c87ac4dbad84cbd4] 2024-11-22T03:41:38,318 INFO [RS:0;c85114ed5096:42545-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 47b13a1c78111d05c87ac4dbad84cbd4#B#compaction#301 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 53.85 MB/second 2024-11-22T03:41:38,319 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/.tmp/B/8a6f7901af734d63a36cc32d5b681ca3 is 50, key is test_row_0/B:col10/1732246896374/Put/seqid=0 2024-11-22T03:41:38,327 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241122bb50532692d94b9aa39d421be26fa291_47b13a1c78111d05c87ac4dbad84cbd4 store=[table=TestAcidGuarantees family=A region=47b13a1c78111d05c87ac4dbad84cbd4] 2024-11-22T03:41:38,329 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241122bb50532692d94b9aa39d421be26fa291_47b13a1c78111d05c87ac4dbad84cbd4, store=[table=TestAcidGuarantees family=A region=47b13a1c78111d05c87ac4dbad84cbd4] 2024-11-22T03:41:38,329 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241122bb50532692d94b9aa39d421be26fa291_47b13a1c78111d05c87ac4dbad84cbd4 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=47b13a1c78111d05c87ac4dbad84cbd4] 2024-11-22T03:41:38,335 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742178_1354 (size=12663) 2024-11-22T03:41:38,339 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742179_1355 (size=4469) 2024-11-22T03:41:38,341 INFO [RS:0;c85114ed5096:42545-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 47b13a1c78111d05c87ac4dbad84cbd4#A#compaction#300 average throughput is 0.98 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-22T03:41:38,342 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/.tmp/A/16ac67e2790f4d9f860c5ef9171b7752 is 175, key is test_row_0/A:col10/1732246896374/Put/seqid=0 2024-11-22T03:41:38,366 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742180_1356 (size=31617) 2024-11-22T03:41:38,371 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/.tmp/A/16ac67e2790f4d9f860c5ef9171b7752 as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/A/16ac67e2790f4d9f860c5ef9171b7752 2024-11-22T03:41:38,378 INFO [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 47b13a1c78111d05c87ac4dbad84cbd4/A of 47b13a1c78111d05c87ac4dbad84cbd4 into 16ac67e2790f4d9f860c5ef9171b7752(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T03:41:38,378 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 47b13a1c78111d05c87ac4dbad84cbd4: 2024-11-22T03:41:38,378 INFO [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., storeName=47b13a1c78111d05c87ac4dbad84cbd4/A, priority=13, startTime=1732246898297; duration=0sec 2024-11-22T03:41:38,378 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T03:41:38,378 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 47b13a1c78111d05c87ac4dbad84cbd4:A 2024-11-22T03:41:38,378 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T03:41:38,381 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T03:41:38,381 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.HStore(1541): 47b13a1c78111d05c87ac4dbad84cbd4/C is initiating minor compaction (all files) 2024-11-22T03:41:38,381 INFO [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 47b13a1c78111d05c87ac4dbad84cbd4/C in TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4. 2024-11-22T03:41:38,381 INFO [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/C/e476a4c625644afc8f837f4f8ec07741, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/C/19e2207df0f44c29bb29a3fca70a6791, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/C/516c3d5a186e4d159a071993f4ca8771] into tmpdir=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/.tmp, totalSize=36.0 K 2024-11-22T03:41:38,382 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] compactions.Compactor(225): Compacting e476a4c625644afc8f837f4f8ec07741, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1732246892941 2024-11-22T03:41:38,382 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] compactions.Compactor(225): Compacting 19e2207df0f44c29bb29a3fca70a6791, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=196, earliestPutTs=1732246894180 2024-11-22T03:41:38,382 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] compactions.Compactor(225): Compacting 516c3d5a186e4d159a071993f4ca8771, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=212, earliestPutTs=1732246896364 2024-11-22T03:41:38,394 INFO [RS:0;c85114ed5096:42545-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 47b13a1c78111d05c87ac4dbad84cbd4#C#compaction#302 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-22T03:41:38,395 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/.tmp/C/29af4193c9384085bf01ff57b14e4402 is 50, key is test_row_0/C:col10/1732246896374/Put/seqid=0 2024-11-22T03:41:38,413 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742181_1357 (size=12663) 2024-11-22T03:41:38,423 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/.tmp/C/29af4193c9384085bf01ff57b14e4402 as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/C/29af4193c9384085bf01ff57b14e4402 2024-11-22T03:41:38,431 INFO [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 47b13a1c78111d05c87ac4dbad84cbd4/C of 47b13a1c78111d05c87ac4dbad84cbd4 into 29af4193c9384085bf01ff57b14e4402(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T03:41:38,431 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 47b13a1c78111d05c87ac4dbad84cbd4: 2024-11-22T03:41:38,431 INFO [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., storeName=47b13a1c78111d05c87ac4dbad84cbd4/C, priority=13, startTime=1732246898297; duration=0sec 2024-11-22T03:41:38,431 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T03:41:38,431 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 47b13a1c78111d05c87ac4dbad84cbd4:C 2024-11-22T03:41:38,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-22T03:41:38,558 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-22T03:41:38,559 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42627 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-22T03:41:38,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42627 {}] procedure2.ProcedureExecutor(1139): Stored pid=103, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees 2024-11-22T03:41:38,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-22T03:41:38,561 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=103, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-22T03:41:38,561 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=103, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-22T03:41:38,561 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=104, ppid=103, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-22T03:41:38,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-22T03:41:38,713 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42545 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-22T03:41:38,713 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4. 2024-11-22T03:41:38,713 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2902): Flushing 47b13a1c78111d05c87ac4dbad84cbd4 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-22T03:41:38,718 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112244dc6e625f2845a5b63a15a7dfbe2ea5_47b13a1c78111d05c87ac4dbad84cbd4 is 50, key is test_row_0/A:col10/1732246897077/Put/seqid=0 2024-11-22T03:41:38,721 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742182_1358 (size=12304) 2024-11-22T03:41:38,739 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/.tmp/B/8a6f7901af734d63a36cc32d5b681ca3 as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/B/8a6f7901af734d63a36cc32d5b681ca3 2024-11-22T03:41:38,743 INFO [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 47b13a1c78111d05c87ac4dbad84cbd4/B of 47b13a1c78111d05c87ac4dbad84cbd4 into 8a6f7901af734d63a36cc32d5b681ca3(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T03:41:38,743 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 47b13a1c78111d05c87ac4dbad84cbd4: 2024-11-22T03:41:38,743 INFO [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., storeName=47b13a1c78111d05c87ac4dbad84cbd4/B, priority=13, startTime=1732246898297; duration=0sec 2024-11-22T03:41:38,743 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T03:41:38,743 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 47b13a1c78111d05c87ac4dbad84cbd4:B 2024-11-22T03:41:38,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-22T03:41:39,122 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:39,125 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112244dc6e625f2845a5b63a15a7dfbe2ea5_47b13a1c78111d05c87ac4dbad84cbd4 to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112244dc6e625f2845a5b63a15a7dfbe2ea5_47b13a1c78111d05c87ac4dbad84cbd4 2024-11-22T03:41:39,125 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/.tmp/A/3ca0ad1a50e548088016069e85920764, store: [table=TestAcidGuarantees family=A region=47b13a1c78111d05c87ac4dbad84cbd4] 2024-11-22T03:41:39,126 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/.tmp/A/3ca0ad1a50e548088016069e85920764 is 175, key is test_row_0/A:col10/1732246897077/Put/seqid=0 2024-11-22T03:41:39,136 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742183_1359 (size=31105) 2024-11-22T03:41:39,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-22T03:41:39,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(8855): Flush requested on 47b13a1c78111d05c87ac4dbad84cbd4 2024-11-22T03:41:39,274 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4. as already flushing 2024-11-22T03:41:39,296 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:39,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52478 deadline: 1732246959289, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:39,296 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:39,296 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:39,296 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:41:39,299 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:39,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52486 deadline: 1732246959292, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:39,299 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:39,300 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:39,300 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:41:39,302 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:39,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52458 deadline: 1732246959296, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:39,302 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:39,302 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:39,302 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:41:39,400 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:39,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52478 deadline: 1732246959399, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:39,400 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:39,400 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:39,400 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:41:39,413 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:39,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52486 deadline: 1732246959408, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:39,413 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:39,413 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:39,413 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:39,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52458 deadline: 1732246959408, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:39,413 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:41:39,413 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:39,413 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:39,413 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:41:39,531 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=237, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/.tmp/A/3ca0ad1a50e548088016069e85920764 2024-11-22T03:41:39,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/.tmp/B/b9f5be7ce49e4954a63880edbd83fc8a is 50, key is test_row_0/B:col10/1732246897077/Put/seqid=0 2024-11-22T03:41:39,543 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742184_1360 (size=12151) 2024-11-22T03:41:39,544 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=237 (bloomFilter=true), to=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/.tmp/B/b9f5be7ce49e4954a63880edbd83fc8a 2024-11-22T03:41:39,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/.tmp/C/6251b861806c4b2f8d8db7cff1a7d194 is 50, key is test_row_0/C:col10/1732246897077/Put/seqid=0 2024-11-22T03:41:39,562 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742185_1361 (size=12151) 2024-11-22T03:41:39,562 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=237 (bloomFilter=true), to=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/.tmp/C/6251b861806c4b2f8d8db7cff1a7d194 2024-11-22T03:41:39,570 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/.tmp/A/3ca0ad1a50e548088016069e85920764 as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/A/3ca0ad1a50e548088016069e85920764 2024-11-22T03:41:39,573 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/A/3ca0ad1a50e548088016069e85920764, entries=150, sequenceid=237, filesize=30.4 K 2024-11-22T03:41:39,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/.tmp/B/b9f5be7ce49e4954a63880edbd83fc8a as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/B/b9f5be7ce49e4954a63880edbd83fc8a 2024-11-22T03:41:39,579 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/B/b9f5be7ce49e4954a63880edbd83fc8a, entries=150, sequenceid=237, filesize=11.9 K 2024-11-22T03:41:39,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/.tmp/C/6251b861806c4b2f8d8db7cff1a7d194 as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/C/6251b861806c4b2f8d8db7cff1a7d194 2024-11-22T03:41:39,585 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/C/6251b861806c4b2f8d8db7cff1a7d194, entries=150, sequenceid=237, filesize=11.9 K 2024-11-22T03:41:39,586 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for 47b13a1c78111d05c87ac4dbad84cbd4 in 873ms, sequenceid=237, compaction requested=false 2024-11-22T03:41:39,586 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2603): Flush status journal for 47b13a1c78111d05c87ac4dbad84cbd4: 2024-11-22T03:41:39,586 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4. 2024-11-22T03:41:39,586 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=104 2024-11-22T03:41:39,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42627 {}] master.HMaster(4169): Remote procedure done, pid=104 2024-11-22T03:41:39,588 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=104, resume processing ppid=103 2024-11-22T03:41:39,588 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=104, ppid=103, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0260 sec 2024-11-22T03:41:39,590 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=103, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees in 1.0300 sec 2024-11-22T03:41:39,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(8855): Flush requested on 47b13a1c78111d05c87ac4dbad84cbd4 2024-11-22T03:41:39,610 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-22T03:41:39,610 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 47b13a1c78111d05c87ac4dbad84cbd4 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-22T03:41:39,620 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112225c32beb1db649f3afd1de8cf2090e12_47b13a1c78111d05c87ac4dbad84cbd4 is 50, key is test_row_0/A:col10/1732246899609/Put/seqid=0 2024-11-22T03:41:39,634 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742186_1362 (size=14794) 2024-11-22T03:41:39,642 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:39,645 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112225c32beb1db649f3afd1de8cf2090e12_47b13a1c78111d05c87ac4dbad84cbd4 to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112225c32beb1db649f3afd1de8cf2090e12_47b13a1c78111d05c87ac4dbad84cbd4 2024-11-22T03:41:39,648 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/.tmp/A/4c80da1635604d1f803e0e1977bdcbcc, store: [table=TestAcidGuarantees family=A region=47b13a1c78111d05c87ac4dbad84cbd4] 2024-11-22T03:41:39,649 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/.tmp/A/4c80da1635604d1f803e0e1977bdcbcc is 175, key is test_row_0/A:col10/1732246899609/Put/seqid=0 2024-11-22T03:41:39,666 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742187_1363 (size=39749) 2024-11-22T03:41:39,667 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=252, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/.tmp/A/4c80da1635604d1f803e0e1977bdcbcc 2024-11-22T03:41:39,674 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/.tmp/B/f7df388d64774bd3abfbd9ffa291111b is 50, key is test_row_0/B:col10/1732246899609/Put/seqid=0 2024-11-22T03:41:39,680 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742188_1364 (size=12151) 2024-11-22T03:41:39,696 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:39,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52486 deadline: 1732246959684, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:39,697 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:39,697 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:39,697 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:41:39,697 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:39,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52478 deadline: 1732246959684, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:39,697 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:39,697 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:39,697 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:41:39,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-22T03:41:39,699 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-22T03:41:39,699 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42627 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-22T03:41:39,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42627 {}] procedure2.ProcedureExecutor(1139): Stored pid=105, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees 2024-11-22T03:41:39,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-22T03:41:39,701 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=105, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-22T03:41:39,702 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=105, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-22T03:41:39,702 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=106, ppid=105, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-22T03:41:39,709 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:39,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52458 deadline: 1732246959697, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:39,709 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:39,709 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:39,709 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:41:39,801 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:39,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52486 deadline: 1732246959799, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:39,801 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:39,801 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:39,801 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:41:39,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-22T03:41:39,813 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:39,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52478 deadline: 1732246959809, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:39,814 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:39,814 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:39,814 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:41:39,825 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:39,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52458 deadline: 1732246959818, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:39,826 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:39,826 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:39,826 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:41:39,853 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42545 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-22T03:41:39,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4. 2024-11-22T03:41:39,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4. as already flushing 2024-11-22T03:41:39,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4. 2024-11-22T03:41:39,854 ERROR [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 47b13a1c78111d05c87ac4dbad84cbd4, NAME => 'TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:41:39,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 47b13a1c78111d05c87ac4dbad84cbd4, NAME => 'TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:41:39,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42627 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 47b13a1c78111d05c87ac4dbad84cbd4, NAME => 'TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 47b13a1c78111d05c87ac4dbad84cbd4, NAME => 'TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:41:40,006 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42545 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-22T03:41:40,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4. 2024-11-22T03:41:40,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4. as already flushing 2024-11-22T03:41:40,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4. 2024-11-22T03:41:40,006 ERROR [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 47b13a1c78111d05c87ac4dbad84cbd4, NAME => 'TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:41:40,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 47b13a1c78111d05c87ac4dbad84cbd4, NAME => 'TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:41:40,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42627 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 47b13a1c78111d05c87ac4dbad84cbd4, NAME => 'TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 47b13a1c78111d05c87ac4dbad84cbd4, NAME => 'TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:41:40,015 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:40,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52486 deadline: 1732246960009, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:40,016 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:40,016 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:40,016 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:41:40,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-22T03:41:40,028 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:40,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52478 deadline: 1732246960019, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:40,028 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:40,028 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:40,028 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:41:40,035 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:40,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52458 deadline: 1732246960028, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:40,036 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:40,036 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:40,036 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:41:40,080 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=252 (bloomFilter=true), to=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/.tmp/B/f7df388d64774bd3abfbd9ffa291111b 2024-11-22T03:41:40,086 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/.tmp/C/9227a1c151cb477ea8c5ff41fe46941e is 50, key is test_row_0/C:col10/1732246899609/Put/seqid=0 2024-11-22T03:41:40,092 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742189_1365 (size=12151) 2024-11-22T03:41:40,094 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=252 (bloomFilter=true), to=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/.tmp/C/9227a1c151cb477ea8c5ff41fe46941e 2024-11-22T03:41:40,099 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/.tmp/A/4c80da1635604d1f803e0e1977bdcbcc as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/A/4c80da1635604d1f803e0e1977bdcbcc 2024-11-22T03:41:40,102 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/A/4c80da1635604d1f803e0e1977bdcbcc, entries=200, sequenceid=252, filesize=38.8 K 2024-11-22T03:41:40,102 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/.tmp/B/f7df388d64774bd3abfbd9ffa291111b as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/B/f7df388d64774bd3abfbd9ffa291111b 2024-11-22T03:41:40,106 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/B/f7df388d64774bd3abfbd9ffa291111b, entries=150, sequenceid=252, filesize=11.9 K 2024-11-22T03:41:40,106 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/.tmp/C/9227a1c151cb477ea8c5ff41fe46941e as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/C/9227a1c151cb477ea8c5ff41fe46941e 2024-11-22T03:41:40,109 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/C/9227a1c151cb477ea8c5ff41fe46941e, entries=150, sequenceid=252, filesize=11.9 K 2024-11-22T03:41:40,110 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for 47b13a1c78111d05c87ac4dbad84cbd4 in 500ms, sequenceid=252, compaction requested=true 2024-11-22T03:41:40,110 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 47b13a1c78111d05c87ac4dbad84cbd4: 2024-11-22T03:41:40,110 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 47b13a1c78111d05c87ac4dbad84cbd4:A, priority=-2147483648, current under compaction store size is 1 2024-11-22T03:41:40,110 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T03:41:40,110 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T03:41:40,110 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T03:41:40,110 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 47b13a1c78111d05c87ac4dbad84cbd4:B, priority=-2147483648, current under compaction store size is 2 2024-11-22T03:41:40,111 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T03:41:40,111 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 47b13a1c78111d05c87ac4dbad84cbd4:C, priority=-2147483648, current under compaction store size is 3 2024-11-22T03:41:40,111 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T03:41:40,113 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102471 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T03:41:40,113 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.HStore(1541): 47b13a1c78111d05c87ac4dbad84cbd4/A is initiating minor compaction (all files) 2024-11-22T03:41:40,113 INFO [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 47b13a1c78111d05c87ac4dbad84cbd4/A in TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4. 2024-11-22T03:41:40,114 INFO [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/A/16ac67e2790f4d9f860c5ef9171b7752, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/A/3ca0ad1a50e548088016069e85920764, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/A/4c80da1635604d1f803e0e1977bdcbcc] into tmpdir=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/.tmp, totalSize=100.1 K 2024-11-22T03:41:40,114 INFO [RS:0;c85114ed5096:42545-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=53.85 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4. 2024-11-22T03:41:40,114 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4. files: [hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/A/16ac67e2790f4d9f860c5ef9171b7752, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/A/3ca0ad1a50e548088016069e85920764, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/A/4c80da1635604d1f803e0e1977bdcbcc] 2024-11-22T03:41:40,114 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] compactions.Compactor(225): Compacting 16ac67e2790f4d9f860c5ef9171b7752, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=212, earliestPutTs=1732246896364 2024-11-22T03:41:40,114 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3ca0ad1a50e548088016069e85920764, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=237, earliestPutTs=1732246897066 2024-11-22T03:41:40,114 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4c80da1635604d1f803e0e1977bdcbcc, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=252, earliestPutTs=1732246899284 2024-11-22T03:41:40,121 INFO [RS:0;c85114ed5096:42545-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=47b13a1c78111d05c87ac4dbad84cbd4] 2024-11-22T03:41:40,121 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36965 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T03:41:40,121 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HStore(1541): 47b13a1c78111d05c87ac4dbad84cbd4/B is initiating minor compaction (all files) 2024-11-22T03:41:40,121 INFO [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 47b13a1c78111d05c87ac4dbad84cbd4/B in TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4. 2024-11-22T03:41:40,121 INFO [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/B/8a6f7901af734d63a36cc32d5b681ca3, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/B/b9f5be7ce49e4954a63880edbd83fc8a, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/B/f7df388d64774bd3abfbd9ffa291111b] into tmpdir=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/.tmp, totalSize=36.1 K 2024-11-22T03:41:40,122 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] compactions.Compactor(225): Compacting 8a6f7901af734d63a36cc32d5b681ca3, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=212, earliestPutTs=1732246896364 2024-11-22T03:41:40,122 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] compactions.Compactor(225): Compacting b9f5be7ce49e4954a63880edbd83fc8a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=237, earliestPutTs=1732246897066 2024-11-22T03:41:40,123 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] compactions.Compactor(225): Compacting f7df388d64774bd3abfbd9ffa291111b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=252, earliestPutTs=1732246899291 2024-11-22T03:41:40,129 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241122b2d22e5266dd4137917c4b564c1afeed_47b13a1c78111d05c87ac4dbad84cbd4 store=[table=TestAcidGuarantees family=A region=47b13a1c78111d05c87ac4dbad84cbd4] 2024-11-22T03:41:40,131 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241122b2d22e5266dd4137917c4b564c1afeed_47b13a1c78111d05c87ac4dbad84cbd4, store=[table=TestAcidGuarantees family=A region=47b13a1c78111d05c87ac4dbad84cbd4] 2024-11-22T03:41:40,131 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241122b2d22e5266dd4137917c4b564c1afeed_47b13a1c78111d05c87ac4dbad84cbd4 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=47b13a1c78111d05c87ac4dbad84cbd4] 2024-11-22T03:41:40,139 INFO [RS:0;c85114ed5096:42545-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 47b13a1c78111d05c87ac4dbad84cbd4#B#compaction#310 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 53.85 MB/second 2024-11-22T03:41:40,140 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/.tmp/B/e738aa7a0a4640f18c11a7d09571de4b is 50, key is test_row_0/B:col10/1732246899609/Put/seqid=0 2024-11-22T03:41:40,140 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742190_1366 (size=4469) 2024-11-22T03:41:40,143 INFO [RS:0;c85114ed5096:42545-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 47b13a1c78111d05c87ac4dbad84cbd4#A#compaction#309 average throughput is 1.11 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-22T03:41:40,143 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/.tmp/A/87bbeed4cbf049799a3bf7c69f85d06e is 175, key is test_row_0/A:col10/1732246899609/Put/seqid=0 2024-11-22T03:41:40,156 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742191_1367 (size=12765) 2024-11-22T03:41:40,158 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42545 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-22T03:41:40,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4. 2024-11-22T03:41:40,159 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2902): Flushing 47b13a1c78111d05c87ac4dbad84cbd4 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-22T03:41:40,172 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742192_1368 (size=31719) 2024-11-22T03:41:40,177 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/.tmp/A/87bbeed4cbf049799a3bf7c69f85d06e as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/A/87bbeed4cbf049799a3bf7c69f85d06e 2024-11-22T03:41:40,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411220617bd7f18624fda87545997072dea34_47b13a1c78111d05c87ac4dbad84cbd4 is 50, key is test_row_0/A:col10/1732246899681/Put/seqid=0 2024-11-22T03:41:40,188 INFO [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 47b13a1c78111d05c87ac4dbad84cbd4/A of 47b13a1c78111d05c87ac4dbad84cbd4 into 87bbeed4cbf049799a3bf7c69f85d06e(size=31.0 K), total size for store is 31.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T03:41:40,189 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 47b13a1c78111d05c87ac4dbad84cbd4: 2024-11-22T03:41:40,189 INFO [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., storeName=47b13a1c78111d05c87ac4dbad84cbd4/A, priority=13, startTime=1732246900110; duration=0sec 2024-11-22T03:41:40,189 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T03:41:40,189 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 47b13a1c78111d05c87ac4dbad84cbd4:A 2024-11-22T03:41:40,189 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T03:41:40,190 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36965 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T03:41:40,190 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.HStore(1541): 47b13a1c78111d05c87ac4dbad84cbd4/C is initiating minor compaction (all files) 2024-11-22T03:41:40,190 INFO [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 47b13a1c78111d05c87ac4dbad84cbd4/C in TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4. 2024-11-22T03:41:40,190 INFO [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/C/29af4193c9384085bf01ff57b14e4402, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/C/6251b861806c4b2f8d8db7cff1a7d194, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/C/9227a1c151cb477ea8c5ff41fe46941e] into tmpdir=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/.tmp, totalSize=36.1 K 2024-11-22T03:41:40,190 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] compactions.Compactor(225): Compacting 29af4193c9384085bf01ff57b14e4402, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=212, earliestPutTs=1732246896364 2024-11-22T03:41:40,190 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6251b861806c4b2f8d8db7cff1a7d194, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=237, earliestPutTs=1732246897066 2024-11-22T03:41:40,191 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9227a1c151cb477ea8c5ff41fe46941e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=252, earliestPutTs=1732246899291 2024-11-22T03:41:40,215 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742193_1369 (size=12454) 2024-11-22T03:41:40,219 INFO [RS:0;c85114ed5096:42545-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 47b13a1c78111d05c87ac4dbad84cbd4#C#compaction#312 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-22T03:41:40,220 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/.tmp/C/9c06c50f053c4bffb458872d6aae4e3e is 50, key is test_row_0/C:col10/1732246899609/Put/seqid=0 2024-11-22T03:41:40,237 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742194_1370 (size=12765) 2024-11-22T03:41:40,244 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/.tmp/C/9c06c50f053c4bffb458872d6aae4e3e as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/C/9c06c50f053c4bffb458872d6aae4e3e 2024-11-22T03:41:40,253 INFO [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 47b13a1c78111d05c87ac4dbad84cbd4/C of 47b13a1c78111d05c87ac4dbad84cbd4 into 9c06c50f053c4bffb458872d6aae4e3e(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T03:41:40,253 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 47b13a1c78111d05c87ac4dbad84cbd4: 2024-11-22T03:41:40,253 INFO [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., storeName=47b13a1c78111d05c87ac4dbad84cbd4/C, priority=13, startTime=1732246900111; duration=0sec 2024-11-22T03:41:40,253 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T03:41:40,253 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 47b13a1c78111d05c87ac4dbad84cbd4:C 2024-11-22T03:41:40,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(8855): Flush requested on 47b13a1c78111d05c87ac4dbad84cbd4 2024-11-22T03:41:40,322 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4. as already flushing 2024-11-22T03:41:40,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-22T03:41:40,389 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:40,389 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:40,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52478 deadline: 1732246960378, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:40,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52486 deadline: 1732246960379, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:40,389 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:40,389 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:40,389 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:41:40,390 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:40,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52458 deadline: 1732246960380, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:40,390 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:40,390 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:40,390 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:41:40,390 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:40,390 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:40,390 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:41:40,504 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:40,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52478 deadline: 1732246960498, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:40,505 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:40,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52458 deadline: 1732246960498, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:40,505 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:40,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52486 deadline: 1732246960498, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:40,505 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:40,505 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:40,505 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:41:40,505 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:40,505 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:40,505 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:40,505 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:40,505 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:41:40,505 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:41:40,560 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/.tmp/B/e738aa7a0a4640f18c11a7d09571de4b as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/B/e738aa7a0a4640f18c11a7d09571de4b 2024-11-22T03:41:40,564 INFO [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 47b13a1c78111d05c87ac4dbad84cbd4/B of 47b13a1c78111d05c87ac4dbad84cbd4 into e738aa7a0a4640f18c11a7d09571de4b(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T03:41:40,564 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 47b13a1c78111d05c87ac4dbad84cbd4: 2024-11-22T03:41:40,564 INFO [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., storeName=47b13a1c78111d05c87ac4dbad84cbd4/B, priority=13, startTime=1732246900110; duration=0sec 2024-11-22T03:41:40,564 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T03:41:40,564 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 47b13a1c78111d05c87ac4dbad84cbd4:B 2024-11-22T03:41:40,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:40,618 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411220617bd7f18624fda87545997072dea34_47b13a1c78111d05c87ac4dbad84cbd4 to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411220617bd7f18624fda87545997072dea34_47b13a1c78111d05c87ac4dbad84cbd4 2024-11-22T03:41:40,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/.tmp/A/5ff59f4d55a14a669b5019a0332ea1e5, store: [table=TestAcidGuarantees family=A region=47b13a1c78111d05c87ac4dbad84cbd4] 2024-11-22T03:41:40,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/.tmp/A/5ff59f4d55a14a669b5019a0332ea1e5 is 175, key is test_row_0/A:col10/1732246899681/Put/seqid=0 2024-11-22T03:41:40,630 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742195_1371 (size=31255) 2024-11-22T03:41:40,710 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:40,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52458 deadline: 1732246960708, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:40,710 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:40,710 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:40,711 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:40,711 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:41:40,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52486 deadline: 1732246960709, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:40,711 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:40,711 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:40,711 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:40,711 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:41:40,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52478 deadline: 1732246960709, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:40,711 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:40,711 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:40,711 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:41:40,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-22T03:41:41,025 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:41,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52458 deadline: 1732246961018, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:41,026 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:41,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52478 deadline: 1732246961018, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:41,026 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:41,026 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:41,026 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:41:41,026 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:41,026 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:41,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52486 deadline: 1732246961018, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:41,026 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:41,026 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:41:41,026 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:41,026 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:41,026 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:41:41,030 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=274, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/.tmp/A/5ff59f4d55a14a669b5019a0332ea1e5 2024-11-22T03:41:41,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/.tmp/B/2b0501aae32140169b7fe29f9f5f06c7 is 50, key is test_row_0/B:col10/1732246899681/Put/seqid=0 2024-11-22T03:41:41,069 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742196_1372 (size=12301) 2024-11-22T03:41:41,071 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=274 (bloomFilter=true), to=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/.tmp/B/2b0501aae32140169b7fe29f9f5f06c7 2024-11-22T03:41:41,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/.tmp/C/e3f8522dbb7a4e7fb0973ec88b8b292a is 50, key is test_row_0/C:col10/1732246899681/Put/seqid=0 2024-11-22T03:41:41,094 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742197_1373 (size=12301) 2024-11-22T03:41:41,495 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=274 (bloomFilter=true), to=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/.tmp/C/e3f8522dbb7a4e7fb0973ec88b8b292a 2024-11-22T03:41:41,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/.tmp/A/5ff59f4d55a14a669b5019a0332ea1e5 as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/A/5ff59f4d55a14a669b5019a0332ea1e5 2024-11-22T03:41:41,501 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/A/5ff59f4d55a14a669b5019a0332ea1e5, entries=150, sequenceid=274, filesize=30.5 K 2024-11-22T03:41:41,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/.tmp/B/2b0501aae32140169b7fe29f9f5f06c7 as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/B/2b0501aae32140169b7fe29f9f5f06c7 2024-11-22T03:41:41,505 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/B/2b0501aae32140169b7fe29f9f5f06c7, entries=150, sequenceid=274, filesize=12.0 K 2024-11-22T03:41:41,506 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/.tmp/C/e3f8522dbb7a4e7fb0973ec88b8b292a as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/C/e3f8522dbb7a4e7fb0973ec88b8b292a 2024-11-22T03:41:41,509 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/C/e3f8522dbb7a4e7fb0973ec88b8b292a, entries=150, sequenceid=274, filesize=12.0 K 2024-11-22T03:41:41,510 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=73.80 KB/75570 for 47b13a1c78111d05c87ac4dbad84cbd4 in 1352ms, sequenceid=274, compaction requested=false 2024-11-22T03:41:41,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2603): Flush status journal for 47b13a1c78111d05c87ac4dbad84cbd4: 2024-11-22T03:41:41,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4. 2024-11-22T03:41:41,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=106 2024-11-22T03:41:41,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42627 {}] master.HMaster(4169): Remote procedure done, pid=106 2024-11-22T03:41:41,513 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=106, resume processing ppid=105 2024-11-22T03:41:41,513 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=106, ppid=105, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8090 sec 2024-11-22T03:41:41,515 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=105, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees in 1.8150 sec 2024-11-22T03:41:41,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(8855): Flush requested on 47b13a1c78111d05c87ac4dbad84cbd4 2024-11-22T03:41:41,534 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-22T03:41:41,534 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 47b13a1c78111d05c87ac4dbad84cbd4 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-22T03:41:41,540 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241122fed56234794943c289e4d61df78c549c_47b13a1c78111d05c87ac4dbad84cbd4 is 50, key is test_row_0/A:col10/1732246900375/Put/seqid=0 2024-11-22T03:41:41,545 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742198_1374 (size=14994) 2024-11-22T03:41:41,549 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:41,552 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241122fed56234794943c289e4d61df78c549c_47b13a1c78111d05c87ac4dbad84cbd4 to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241122fed56234794943c289e4d61df78c549c_47b13a1c78111d05c87ac4dbad84cbd4 2024-11-22T03:41:41,553 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/.tmp/A/30d05496f27844f1ad2d5728d8058aca, store: [table=TestAcidGuarantees family=A region=47b13a1c78111d05c87ac4dbad84cbd4] 2024-11-22T03:41:41,553 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/.tmp/A/30d05496f27844f1ad2d5728d8058aca is 175, key is test_row_0/A:col10/1732246900375/Put/seqid=0 2024-11-22T03:41:41,557 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742199_1375 (size=39949) 2024-11-22T03:41:41,559 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=292, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/.tmp/A/30d05496f27844f1ad2d5728d8058aca 2024-11-22T03:41:41,565 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/.tmp/B/c976218218f040939b42d53cd3946f8e is 50, key is test_row_0/B:col10/1732246900375/Put/seqid=0 2024-11-22T03:41:41,570 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742200_1376 (size=12301) 2024-11-22T03:41:41,571 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=292 (bloomFilter=true), to=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/.tmp/B/c976218218f040939b42d53cd3946f8e 2024-11-22T03:41:41,576 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/.tmp/C/79b0c67ae41d49388d8e367ffc08d5f5 is 50, key is test_row_0/C:col10/1732246900375/Put/seqid=0 2024-11-22T03:41:41,579 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742201_1377 (size=12301) 2024-11-22T03:41:41,597 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:41,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52478 deadline: 1732246961588, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:41,597 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:41,597 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:41,597 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:41:41,597 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:41,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52458 deadline: 1732246961590, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:41,598 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:41,598 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:41,598 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:41:41,599 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:41,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52486 deadline: 1732246961595, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:41,599 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:41,599 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:41,599 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:41:41,713 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:41,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52478 deadline: 1732246961709, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:41,714 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:41,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52458 deadline: 1732246961709, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:41,714 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:41,714 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:41,714 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:41:41,714 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:41,714 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:41,714 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:41,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52486 deadline: 1732246961709, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:41,714 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:41:41,714 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:41,714 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:41,714 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:41:41,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-22T03:41:41,858 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-22T03:41:41,859 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42627 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-22T03:41:41,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42627 {}] procedure2.ProcedureExecutor(1139): Stored pid=107, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees 2024-11-22T03:41:41,860 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=107, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-22T03:41:41,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-22T03:41:41,860 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=107, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-22T03:41:41,860 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=108, ppid=107, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-22T03:41:41,923 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:41,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52478 deadline: 1732246961919, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:41,923 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:41,923 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:41,923 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:41,923 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:41:41,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52458 deadline: 1732246961919, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:41,923 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:41,923 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:41,923 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:41,923 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:41:41,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52486 deadline: 1732246961919, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:41,924 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:41,924 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:41,924 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:41:41,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-22T03:41:41,979 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=292 (bloomFilter=true), to=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/.tmp/C/79b0c67ae41d49388d8e367ffc08d5f5 2024-11-22T03:41:41,983 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/.tmp/A/30d05496f27844f1ad2d5728d8058aca as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/A/30d05496f27844f1ad2d5728d8058aca 2024-11-22T03:41:41,986 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/A/30d05496f27844f1ad2d5728d8058aca, entries=200, sequenceid=292, filesize=39.0 K 2024-11-22T03:41:41,986 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/.tmp/B/c976218218f040939b42d53cd3946f8e as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/B/c976218218f040939b42d53cd3946f8e 2024-11-22T03:41:41,989 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/B/c976218218f040939b42d53cd3946f8e, entries=150, sequenceid=292, filesize=12.0 K 2024-11-22T03:41:41,990 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/.tmp/C/79b0c67ae41d49388d8e367ffc08d5f5 as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/C/79b0c67ae41d49388d8e367ffc08d5f5 2024-11-22T03:41:41,994 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/C/79b0c67ae41d49388d8e367ffc08d5f5, entries=150, sequenceid=292, filesize=12.0 K 2024-11-22T03:41:41,994 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=120.76 KB/123660 for 47b13a1c78111d05c87ac4dbad84cbd4 in 460ms, sequenceid=292, compaction requested=true 2024-11-22T03:41:41,995 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 47b13a1c78111d05c87ac4dbad84cbd4: 2024-11-22T03:41:41,995 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T03:41:41,995 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 47b13a1c78111d05c87ac4dbad84cbd4:A, priority=-2147483648, current under compaction store size is 1 2024-11-22T03:41:41,995 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T03:41:41,995 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T03:41:41,995 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 47b13a1c78111d05c87ac4dbad84cbd4:B, priority=-2147483648, current under compaction store size is 2 2024-11-22T03:41:41,995 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T03:41:41,995 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 47b13a1c78111d05c87ac4dbad84cbd4:C, priority=-2147483648, current under compaction store size is 3 2024-11-22T03:41:41,995 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T03:41:41,997 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37367 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T03:41:41,997 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102923 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T03:41:41,997 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.HStore(1541): 47b13a1c78111d05c87ac4dbad84cbd4/A is initiating minor compaction (all files) 2024-11-22T03:41:41,997 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HStore(1541): 47b13a1c78111d05c87ac4dbad84cbd4/B is initiating minor compaction (all files) 2024-11-22T03:41:41,997 INFO [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 47b13a1c78111d05c87ac4dbad84cbd4/B in TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4. 2024-11-22T03:41:41,997 INFO [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 47b13a1c78111d05c87ac4dbad84cbd4/A in TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4. 2024-11-22T03:41:41,998 INFO [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/B/e738aa7a0a4640f18c11a7d09571de4b, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/B/2b0501aae32140169b7fe29f9f5f06c7, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/B/c976218218f040939b42d53cd3946f8e] into tmpdir=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/.tmp, totalSize=36.5 K 2024-11-22T03:41:41,998 INFO [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/A/87bbeed4cbf049799a3bf7c69f85d06e, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/A/5ff59f4d55a14a669b5019a0332ea1e5, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/A/30d05496f27844f1ad2d5728d8058aca] into tmpdir=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/.tmp, totalSize=100.5 K 2024-11-22T03:41:41,998 INFO [RS:0;c85114ed5096:42545-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=53.85 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4. 2024-11-22T03:41:41,998 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4. files: [hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/A/87bbeed4cbf049799a3bf7c69f85d06e, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/A/5ff59f4d55a14a669b5019a0332ea1e5, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/A/30d05496f27844f1ad2d5728d8058aca] 2024-11-22T03:41:41,998 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] compactions.Compactor(225): Compacting e738aa7a0a4640f18c11a7d09571de4b, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=252, earliestPutTs=1732246899291 2024-11-22T03:41:41,998 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] compactions.Compactor(225): Compacting 2b0501aae32140169b7fe29f9f5f06c7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=274, earliestPutTs=1732246899681 2024-11-22T03:41:41,999 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] compactions.Compactor(225): Compacting 87bbeed4cbf049799a3bf7c69f85d06e, keycount=150, bloomtype=ROW, size=31.0 K, encoding=NONE, compression=NONE, seqNum=252, earliestPutTs=1732246899291 2024-11-22T03:41:41,999 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] compactions.Compactor(225): Compacting c976218218f040939b42d53cd3946f8e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=292, earliestPutTs=1732246900375 2024-11-22T03:41:41,999 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5ff59f4d55a14a669b5019a0332ea1e5, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=274, earliestPutTs=1732246899681 2024-11-22T03:41:41,999 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] compactions.Compactor(225): Compacting 30d05496f27844f1ad2d5728d8058aca, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=292, earliestPutTs=1732246900375 2024-11-22T03:41:42,006 INFO [RS:0;c85114ed5096:42545-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=47b13a1c78111d05c87ac4dbad84cbd4] 2024-11-22T03:41:42,007 INFO [RS:0;c85114ed5096:42545-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 47b13a1c78111d05c87ac4dbad84cbd4#B#compaction#319 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 53.85 MB/second 2024-11-22T03:41:42,007 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241122ce8970c9c1d8443eaeade01d608daebf_47b13a1c78111d05c87ac4dbad84cbd4 store=[table=TestAcidGuarantees family=A region=47b13a1c78111d05c87ac4dbad84cbd4] 2024-11-22T03:41:42,008 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/.tmp/B/943c0c5eb2b94c4d9012fd5cb9f3a730 is 50, key is test_row_0/B:col10/1732246900375/Put/seqid=0 2024-11-22T03:41:42,009 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241122ce8970c9c1d8443eaeade01d608daebf_47b13a1c78111d05c87ac4dbad84cbd4, store=[table=TestAcidGuarantees family=A region=47b13a1c78111d05c87ac4dbad84cbd4] 2024-11-22T03:41:42,009 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241122ce8970c9c1d8443eaeade01d608daebf_47b13a1c78111d05c87ac4dbad84cbd4 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=47b13a1c78111d05c87ac4dbad84cbd4] 2024-11-22T03:41:42,012 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42545 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-22T03:41:42,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4. 2024-11-22T03:41:42,012 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2902): Flushing 47b13a1c78111d05c87ac4dbad84cbd4 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2024-11-22T03:41:42,033 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742202_1378 (size=13017) 2024-11-22T03:41:42,034 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742203_1379 (size=4469) 2024-11-22T03:41:42,034 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241122bd08256b4b7c4e6ea0ee8a26f7a94751_47b13a1c78111d05c87ac4dbad84cbd4 is 50, key is test_row_0/A:col10/1732246901594/Put/seqid=0 2024-11-22T03:41:42,042 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742204_1380 (size=12454) 2024-11-22T03:41:42,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(8855): Flush requested on 47b13a1c78111d05c87ac4dbad84cbd4 2024-11-22T03:41:42,062 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4. as already flushing 2024-11-22T03:41:42,117 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:42,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52414 deadline: 1732246962113, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:42,118 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:42,118 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:42,118 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:41:42,121 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:42,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52472 deadline: 1732246962114, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:42,121 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:42,121 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:42,121 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:41:42,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-22T03:41:42,234 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:42,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52478 deadline: 1732246962229, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:42,234 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:42,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52458 deadline: 1732246962229, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:42,234 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:42,234 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:42,234 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:41:42,234 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:42,234 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:42,235 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:42,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52486 deadline: 1732246962229, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:42,235 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:41:42,235 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:42,235 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:42,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52414 deadline: 1732246962229, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:42,235 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:42,235 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:41:42,235 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:42,235 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:42,235 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:42,235 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:41:42,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52472 deadline: 1732246962229, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:42,235 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:42,235 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:42,235 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:41:42,435 INFO [RS:0;c85114ed5096:42545-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 47b13a1c78111d05c87ac4dbad84cbd4#A#compaction#318 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-22T03:41:42,436 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/.tmp/A/2439bbfd5ad9412d80dbec229fd894bb is 175, key is test_row_0/A:col10/1732246900375/Put/seqid=0 2024-11-22T03:41:42,439 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/.tmp/B/943c0c5eb2b94c4d9012fd5cb9f3a730 as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/B/943c0c5eb2b94c4d9012fd5cb9f3a730 2024-11-22T03:41:42,440 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742205_1381 (size=31971) 2024-11-22T03:41:42,441 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:42,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52414 deadline: 1732246962438, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:42,441 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:42,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52472 deadline: 1732246962439, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:42,442 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:42,442 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:42,442 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:41:42,443 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:42,443 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:42,443 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:41:42,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:42,444 INFO [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 47b13a1c78111d05c87ac4dbad84cbd4/B of 47b13a1c78111d05c87ac4dbad84cbd4 into 943c0c5eb2b94c4d9012fd5cb9f3a730(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T03:41:42,444 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 47b13a1c78111d05c87ac4dbad84cbd4: 2024-11-22T03:41:42,444 INFO [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., storeName=47b13a1c78111d05c87ac4dbad84cbd4/B, priority=13, startTime=1732246901995; duration=0sec 2024-11-22T03:41:42,445 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T03:41:42,445 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 47b13a1c78111d05c87ac4dbad84cbd4:B 2024-11-22T03:41:42,445 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T03:41:42,445 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37367 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T03:41:42,445 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HStore(1541): 47b13a1c78111d05c87ac4dbad84cbd4/C is initiating minor compaction (all files) 2024-11-22T03:41:42,445 INFO [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 47b13a1c78111d05c87ac4dbad84cbd4/C in TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4. 2024-11-22T03:41:42,446 INFO [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/C/9c06c50f053c4bffb458872d6aae4e3e, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/C/e3f8522dbb7a4e7fb0973ec88b8b292a, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/C/79b0c67ae41d49388d8e367ffc08d5f5] into tmpdir=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/.tmp, totalSize=36.5 K 2024-11-22T03:41:42,446 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] compactions.Compactor(225): Compacting 9c06c50f053c4bffb458872d6aae4e3e, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=252, earliestPutTs=1732246899291 2024-11-22T03:41:42,446 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/.tmp/A/2439bbfd5ad9412d80dbec229fd894bb as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/A/2439bbfd5ad9412d80dbec229fd894bb 2024-11-22T03:41:42,446 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] compactions.Compactor(225): Compacting e3f8522dbb7a4e7fb0973ec88b8b292a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=274, earliestPutTs=1732246899681 2024-11-22T03:41:42,446 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241122bd08256b4b7c4e6ea0ee8a26f7a94751_47b13a1c78111d05c87ac4dbad84cbd4 to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241122bd08256b4b7c4e6ea0ee8a26f7a94751_47b13a1c78111d05c87ac4dbad84cbd4 2024-11-22T03:41:42,447 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] compactions.Compactor(225): Compacting 79b0c67ae41d49388d8e367ffc08d5f5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=292, earliestPutTs=1732246900375 2024-11-22T03:41:42,447 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/.tmp/A/f8af0228d5164e5c81870244e1412d40, store: [table=TestAcidGuarantees family=A region=47b13a1c78111d05c87ac4dbad84cbd4] 2024-11-22T03:41:42,447 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/.tmp/A/f8af0228d5164e5c81870244e1412d40 is 175, key is test_row_0/A:col10/1732246901594/Put/seqid=0 2024-11-22T03:41:42,454 INFO [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 47b13a1c78111d05c87ac4dbad84cbd4/A of 47b13a1c78111d05c87ac4dbad84cbd4 into 2439bbfd5ad9412d80dbec229fd894bb(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T03:41:42,454 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 47b13a1c78111d05c87ac4dbad84cbd4: 2024-11-22T03:41:42,454 INFO [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., storeName=47b13a1c78111d05c87ac4dbad84cbd4/A, priority=13, startTime=1732246901995; duration=0sec 2024-11-22T03:41:42,454 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T03:41:42,454 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 47b13a1c78111d05c87ac4dbad84cbd4:A 2024-11-22T03:41:42,462 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742206_1382 (size=31255) 2024-11-22T03:41:42,466 INFO [RS:0;c85114ed5096:42545-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 47b13a1c78111d05c87ac4dbad84cbd4#C#compaction#321 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-22T03:41:42,466 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/.tmp/C/7766eef7d3d248f888df73db4248925a is 50, key is test_row_0/C:col10/1732246900375/Put/seqid=0 2024-11-22T03:41:42,469 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742207_1383 (size=13017) 2024-11-22T03:41:42,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-22T03:41:42,744 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:42,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52478 deadline: 1732246962739, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:42,745 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:42,745 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:42,745 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:42,745 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:41:42,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52458 deadline: 1732246962739, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:42,745 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:42,745 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:42,745 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:41:42,745 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:42,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52486 deadline: 1732246962739, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:42,745 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:42,746 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:42,746 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:41:42,757 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:42,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52414 deadline: 1732246962749, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:42,757 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:42,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52472 deadline: 1732246962749, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:42,758 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:42,758 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:42,758 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:41:42,758 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:42,758 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:42,758 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:41:42,862 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=313, memsize=40.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/.tmp/A/f8af0228d5164e5c81870244e1412d40 2024-11-22T03:41:42,868 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/.tmp/B/d25e75af22ed4441915ae43f3ae7885a is 50, key is test_row_0/B:col10/1732246901594/Put/seqid=0 2024-11-22T03:41:42,874 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/.tmp/C/7766eef7d3d248f888df73db4248925a as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/C/7766eef7d3d248f888df73db4248925a 2024-11-22T03:41:42,875 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742208_1384 (size=12301) 2024-11-22T03:41:42,877 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=313 (bloomFilter=true), to=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/.tmp/B/d25e75af22ed4441915ae43f3ae7885a 2024-11-22T03:41:42,882 INFO [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 47b13a1c78111d05c87ac4dbad84cbd4/C of 47b13a1c78111d05c87ac4dbad84cbd4 into 7766eef7d3d248f888df73db4248925a(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T03:41:42,882 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 47b13a1c78111d05c87ac4dbad84cbd4: 2024-11-22T03:41:42,882 INFO [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., storeName=47b13a1c78111d05c87ac4dbad84cbd4/C, priority=13, startTime=1732246901995; duration=0sec 2024-11-22T03:41:42,882 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T03:41:42,882 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 47b13a1c78111d05c87ac4dbad84cbd4:C 2024-11-22T03:41:42,884 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/.tmp/C/2046c80283f84ba9afb12e68d1e07d3f is 50, key is test_row_0/C:col10/1732246901594/Put/seqid=0 2024-11-22T03:41:42,887 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742209_1385 (size=12301) 2024-11-22T03:41:42,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-22T03:41:43,275 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:43,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52414 deadline: 1732246963268, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:43,275 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:43,275 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:43,275 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:43,275 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:41:43,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52472 deadline: 1732246963268, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:43,275 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:43,275 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:43,275 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:41:43,287 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=313 (bloomFilter=true), to=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/.tmp/C/2046c80283f84ba9afb12e68d1e07d3f 2024-11-22T03:41:43,291 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/.tmp/A/f8af0228d5164e5c81870244e1412d40 as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/A/f8af0228d5164e5c81870244e1412d40 2024-11-22T03:41:43,294 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/A/f8af0228d5164e5c81870244e1412d40, entries=150, sequenceid=313, filesize=30.5 K 2024-11-22T03:41:43,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/.tmp/B/d25e75af22ed4441915ae43f3ae7885a as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/B/d25e75af22ed4441915ae43f3ae7885a 2024-11-22T03:41:43,301 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/B/d25e75af22ed4441915ae43f3ae7885a, entries=150, sequenceid=313, filesize=12.0 K 2024-11-22T03:41:43,301 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/.tmp/C/2046c80283f84ba9afb12e68d1e07d3f as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/C/2046c80283f84ba9afb12e68d1e07d3f 2024-11-22T03:41:43,304 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/C/2046c80283f84ba9afb12e68d1e07d3f, entries=150, sequenceid=313, filesize=12.0 K 2024-11-22T03:41:43,307 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~317.11 KB/324720, currentSize=80.51 KB/82440 for 47b13a1c78111d05c87ac4dbad84cbd4 in 1295ms, sequenceid=313, compaction requested=false 2024-11-22T03:41:43,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2603): Flush status journal for 47b13a1c78111d05c87ac4dbad84cbd4: 2024-11-22T03:41:43,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4. 2024-11-22T03:41:43,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=108 2024-11-22T03:41:43,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42627 {}] master.HMaster(4169): Remote procedure done, pid=108 2024-11-22T03:41:43,311 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=108, resume processing ppid=107 2024-11-22T03:41:43,311 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=108, ppid=107, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4490 sec 2024-11-22T03:41:43,312 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=107, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees in 1.4520 sec 2024-11-22T03:41:43,742 INFO [Thread-1380 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1380. 2024-11-22T03:41:43,742 DEBUG [Thread-1380 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-22T03:41:43,742 DEBUG [Thread-1380 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T03:41:43,742 DEBUG [Thread-1380 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T03:41:43,742 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-22T03:41:43,743 INFO [Thread-1376 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1376. 2024-11-22T03:41:43,743 DEBUG [Thread-1376 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-22T03:41:43,743 DEBUG [Thread-1376 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T03:41:43,743 DEBUG [Thread-1376 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T03:41:43,743 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-22T03:41:43,745 INFO [Thread-1378 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1378. 2024-11-22T03:41:43,745 DEBUG [Thread-1378 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-22T03:41:43,745 DEBUG [Thread-1378 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T03:41:43,745 DEBUG [Thread-1378 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T03:41:43,745 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-22T03:41:43,748 INFO [Thread-1384 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1384. 2024-11-22T03:41:43,748 DEBUG [Thread-1384 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-22T03:41:43,748 DEBUG [Thread-1384 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T03:41:43,748 DEBUG [Thread-1384 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T03:41:43,748 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-22T03:41:43,749 INFO [Thread-1382 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1382. 2024-11-22T03:41:43,749 DEBUG [Thread-1382 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-22T03:41:43,749 DEBUG [Thread-1382 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T03:41:43,749 DEBUG [Thread-1382 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T03:41:43,749 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-22T03:41:43,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(8855): Flush requested on 47b13a1c78111d05c87ac4dbad84cbd4 2024-11-22T03:41:43,761 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-22T03:41:43,761 INFO [Thread-1371 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1371. 2024-11-22T03:41:43,761 INFO [Thread-1369 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1369. 2024-11-22T03:41:43,761 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 47b13a1c78111d05c87ac4dbad84cbd4 3/3 column families, dataSize=100.63 KB heapSize=264.42 KB 2024-11-22T03:41:43,761 DEBUG [Thread-1371 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-22T03:41:43,761 INFO [Thread-1367 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1367. 2024-11-22T03:41:43,761 DEBUG [Thread-1371 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T03:41:43,761 DEBUG [Thread-1369 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-22T03:41:43,761 DEBUG [Thread-1369 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T03:41:43,761 DEBUG [Thread-1367 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-22T03:41:43,761 DEBUG [Thread-1367 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T03:41:43,761 DEBUG [Thread-1371 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T03:41:43,761 DEBUG [Thread-1369 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T03:41:43,761 DEBUG [Thread-1367 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T03:41:43,761 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-22T03:41:43,761 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-22T03:41:43,761 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-22T03:41:43,766 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241122ffb1a02109bc4ba583378aa83244dcf1_47b13a1c78111d05c87ac4dbad84cbd4 is 50, key is test_row_0/A:col10/1732246903759/Put/seqid=0 2024-11-22T03:41:43,769 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742210_1386 (size=12454) 2024-11-22T03:41:44,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-22T03:41:44,009 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-22T03:41:44,171 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:44,180 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241122ffb1a02109bc4ba583378aa83244dcf1_47b13a1c78111d05c87ac4dbad84cbd4 to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241122ffb1a02109bc4ba583378aa83244dcf1_47b13a1c78111d05c87ac4dbad84cbd4 2024-11-22T03:41:44,181 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/.tmp/A/3899fddd1e9c4d2e9c1a54ea1158e121, store: [table=TestAcidGuarantees family=A region=47b13a1c78111d05c87ac4dbad84cbd4] 2024-11-22T03:41:44,182 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/.tmp/A/3899fddd1e9c4d2e9c1a54ea1158e121 is 175, key is test_row_0/A:col10/1732246903759/Put/seqid=0 2024-11-22T03:41:44,185 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742211_1387 (size=31255) 2024-11-22T03:41:44,290 INFO [Thread-1365 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1365. 2024-11-22T03:41:44,290 DEBUG [Thread-1365 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-22T03:41:44,290 INFO [Thread-1373 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1373. 2024-11-22T03:41:44,290 DEBUG [Thread-1365 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T03:41:44,290 DEBUG [Thread-1373 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-22T03:41:44,290 DEBUG [Thread-1373 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T03:41:44,290 DEBUG [Thread-1365 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T03:41:44,290 DEBUG [Thread-1373 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T03:41:44,290 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-22T03:41:44,290 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-22T03:41:44,291 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-22T03:41:44,291 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 17 2024-11-22T03:41:44,291 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 81 2024-11-22T03:41:44,291 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 78 2024-11-22T03:41:44,291 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 75 2024-11-22T03:41:44,291 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 9 2024-11-22T03:41:44,291 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-22T03:41:44,291 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-22T03:41:44,291 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1379 2024-11-22T03:41:44,291 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4133 rows 2024-11-22T03:41:44,291 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1367 2024-11-22T03:41:44,291 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4101 rows 2024-11-22T03:41:44,291 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1374 2024-11-22T03:41:44,291 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4121 rows 2024-11-22T03:41:44,291 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1380 2024-11-22T03:41:44,291 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4133 rows 2024-11-22T03:41:44,291 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1384 2024-11-22T03:41:44,291 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4148 rows 2024-11-22T03:41:44,291 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-22T03:41:44,291 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobScanAtomicity(AcidGuaranteesTestBase.java:127) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-22T03:41:44,291 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T03:41:44,291 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T03:41:44,291 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-22T03:41:44,292 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42627 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-22T03:41:44,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42627 {}] procedure2.ProcedureExecutor(1139): Stored pid=109, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-22T03:41:44,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-22T03:41:44,294 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732246904294"}]},"ts":"1732246904294"} 2024-11-22T03:41:44,296 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-22T03:41:44,296 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-22T03:41:44,297 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=110, ppid=109, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-22T03:41:44,298 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=111, ppid=110, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=47b13a1c78111d05c87ac4dbad84cbd4, UNASSIGN}] 2024-11-22T03:41:44,299 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=111, ppid=110, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=47b13a1c78111d05c87ac4dbad84cbd4, UNASSIGN 2024-11-22T03:41:44,299 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=111 updating hbase:meta row=47b13a1c78111d05c87ac4dbad84cbd4, regionState=CLOSING, regionLocation=c85114ed5096,42545,1732246786482 2024-11-22T03:41:44,301 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=111, ppid=110, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=47b13a1c78111d05c87ac4dbad84cbd4, UNASSIGN because future has completed 2024-11-22T03:41:44,301 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-22T03:41:44,301 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=112, ppid=111, state=RUNNABLE, hasLock=false; CloseRegionProcedure 47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482}] 2024-11-22T03:41:44,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-22T03:41:44,453 INFO [RS_CLOSE_REGION-regionserver/c85114ed5096:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] handler.UnassignRegionHandler(122): Close 47b13a1c78111d05c87ac4dbad84cbd4 2024-11-22T03:41:44,453 DEBUG [RS_CLOSE_REGION-regionserver/c85114ed5096:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-22T03:41:44,453 DEBUG [RS_CLOSE_REGION-regionserver/c85114ed5096:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HRegion(1722): Closing 47b13a1c78111d05c87ac4dbad84cbd4, disabling compactions & flushes 2024-11-22T03:41:44,453 DEBUG [RS_CLOSE_REGION-regionserver/c85114ed5096:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HRegion(1993): waiting for 0 compactions & cache flush to complete for region TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4. 2024-11-22T03:41:44,586 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=334, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/.tmp/A/3899fddd1e9c4d2e9c1a54ea1158e121 2024-11-22T03:41:44,596 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/.tmp/B/2f1e2ed457ab4c44aad5c41664f4e5ea is 50, key is test_row_0/B:col10/1732246903759/Put/seqid=0 2024-11-22T03:41:44,602 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742212_1388 (size=12301) 2024-11-22T03:41:44,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-22T03:41:44,823 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-22T03:41:44,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-22T03:41:45,002 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=334 (bloomFilter=true), to=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/.tmp/B/2f1e2ed457ab4c44aad5c41664f4e5ea 2024-11-22T03:41:45,008 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/.tmp/C/d43365d32aac4cd0801ec0753618022e is 50, key is test_row_0/C:col10/1732246903759/Put/seqid=0 2024-11-22T03:41:45,012 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742213_1389 (size=12301) 2024-11-22T03:41:45,412 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=334 (bloomFilter=true), to=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/.tmp/C/d43365d32aac4cd0801ec0753618022e 2024-11-22T03:41:45,416 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/.tmp/A/3899fddd1e9c4d2e9c1a54ea1158e121 as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/A/3899fddd1e9c4d2e9c1a54ea1158e121 2024-11-22T03:41:45,419 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/A/3899fddd1e9c4d2e9c1a54ea1158e121, entries=150, sequenceid=334, filesize=30.5 K 2024-11-22T03:41:45,420 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/.tmp/B/2f1e2ed457ab4c44aad5c41664f4e5ea as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/B/2f1e2ed457ab4c44aad5c41664f4e5ea 2024-11-22T03:41:45,423 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/B/2f1e2ed457ab4c44aad5c41664f4e5ea, entries=150, sequenceid=334, filesize=12.0 K 2024-11-22T03:41:45,424 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/.tmp/C/d43365d32aac4cd0801ec0753618022e as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/C/d43365d32aac4cd0801ec0753618022e 2024-11-22T03:41:45,426 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/C/d43365d32aac4cd0801ec0753618022e, entries=150, sequenceid=334, filesize=12.0 K 2024-11-22T03:41:45,427 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~264.38 KB/270720, currentSize=13.42 KB/13740 for 47b13a1c78111d05c87ac4dbad84cbd4 in 1666ms, sequenceid=334, compaction requested=true 2024-11-22T03:41:45,427 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 47b13a1c78111d05c87ac4dbad84cbd4: 2024-11-22T03:41:45,427 INFO [RS_CLOSE_REGION-regionserver/c85114ed5096:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4. 2024-11-22T03:41:45,427 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 47b13a1c78111d05c87ac4dbad84cbd4:A, priority=-2147483648, current under compaction store size is 1 2024-11-22T03:41:45,427 DEBUG [RS_CLOSE_REGION-regionserver/c85114ed5096:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4. 2024-11-22T03:41:45,427 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T03:41:45,427 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4. because compaction request was cancelled 2024-11-22T03:41:45,427 DEBUG [RS_CLOSE_REGION-regionserver/c85114ed5096:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4. after waiting 0 ms 2024-11-22T03:41:45,427 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 47b13a1c78111d05c87ac4dbad84cbd4:B, priority=-2147483648, current under compaction store size is 2 2024-11-22T03:41:45,427 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 47b13a1c78111d05c87ac4dbad84cbd4:A 2024-11-22T03:41:45,427 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4. because compaction request was cancelled 2024-11-22T03:41:45,427 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T03:41:45,427 DEBUG [RS_CLOSE_REGION-regionserver/c85114ed5096:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4. 2024-11-22T03:41:45,427 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 47b13a1c78111d05c87ac4dbad84cbd4:B 2024-11-22T03:41:45,427 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 47b13a1c78111d05c87ac4dbad84cbd4:C, priority=-2147483648, current under compaction store size is 2 2024-11-22T03:41:45,427 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T03:41:45,427 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4. because compaction request was cancelled 2024-11-22T03:41:45,427 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 47b13a1c78111d05c87ac4dbad84cbd4:C 2024-11-22T03:41:45,427 INFO [RS_CLOSE_REGION-regionserver/c85114ed5096:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HRegion(2902): Flushing 47b13a1c78111d05c87ac4dbad84cbd4 3/3 column families, dataSize=13.42 KB heapSize=35.91 KB 2024-11-22T03:41:45,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-22T03:41:45,432 DEBUG [RS_CLOSE_REGION-regionserver/c85114ed5096:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411228c3bcea1cc3641bdaddb6b2f10b35260_47b13a1c78111d05c87ac4dbad84cbd4 is 50, key is test_row_1/A:col10/1732246904289/Put/seqid=0 2024-11-22T03:41:45,437 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742214_1390 (size=7374) 2024-11-22T03:41:45,837 DEBUG [RS_CLOSE_REGION-regionserver/c85114ed5096:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:45,841 INFO [RS_CLOSE_REGION-regionserver/c85114ed5096:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411228c3bcea1cc3641bdaddb6b2f10b35260_47b13a1c78111d05c87ac4dbad84cbd4 to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411228c3bcea1cc3641bdaddb6b2f10b35260_47b13a1c78111d05c87ac4dbad84cbd4 2024-11-22T03:41:45,842 DEBUG [RS_CLOSE_REGION-regionserver/c85114ed5096:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/.tmp/A/5088a2ac058f4963b79b71d9e91654b3, store: [table=TestAcidGuarantees family=A region=47b13a1c78111d05c87ac4dbad84cbd4] 2024-11-22T03:41:45,842 DEBUG [RS_CLOSE_REGION-regionserver/c85114ed5096:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/.tmp/A/5088a2ac058f4963b79b71d9e91654b3 is 175, key is test_row_1/A:col10/1732246904289/Put/seqid=0 2024-11-22T03:41:45,845 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742215_1391 (size=13865) 2024-11-22T03:41:46,246 INFO [RS_CLOSE_REGION-regionserver/c85114ed5096:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=339, memsize=4.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/.tmp/A/5088a2ac058f4963b79b71d9e91654b3 2024-11-22T03:41:46,251 DEBUG [RS_CLOSE_REGION-regionserver/c85114ed5096:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/.tmp/B/4f3be9bfbee34ffaad735dfbcea9b838 is 50, key is test_row_1/B:col10/1732246904289/Put/seqid=0 2024-11-22T03:41:46,253 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742216_1392 (size=7415) 2024-11-22T03:41:46,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-22T03:41:46,654 INFO [RS_CLOSE_REGION-regionserver/c85114ed5096:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=339 (bloomFilter=true), to=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/.tmp/B/4f3be9bfbee34ffaad735dfbcea9b838 2024-11-22T03:41:46,659 DEBUG [RS_CLOSE_REGION-regionserver/c85114ed5096:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/.tmp/C/00c9f28186fb4ae3a7dfb5b58bee9d49 is 50, key is test_row_1/C:col10/1732246904289/Put/seqid=0 2024-11-22T03:41:46,662 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742217_1393 (size=7415) 2024-11-22T03:41:47,063 INFO [RS_CLOSE_REGION-regionserver/c85114ed5096:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=339 (bloomFilter=true), to=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/.tmp/C/00c9f28186fb4ae3a7dfb5b58bee9d49 2024-11-22T03:41:47,073 DEBUG [RS_CLOSE_REGION-regionserver/c85114ed5096:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/.tmp/A/5088a2ac058f4963b79b71d9e91654b3 as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/A/5088a2ac058f4963b79b71d9e91654b3 2024-11-22T03:41:47,078 INFO [RS_CLOSE_REGION-regionserver/c85114ed5096:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/A/5088a2ac058f4963b79b71d9e91654b3, entries=50, sequenceid=339, filesize=13.5 K 2024-11-22T03:41:47,079 DEBUG [RS_CLOSE_REGION-regionserver/c85114ed5096:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/.tmp/B/4f3be9bfbee34ffaad735dfbcea9b838 as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/B/4f3be9bfbee34ffaad735dfbcea9b838 2024-11-22T03:41:47,083 INFO [RS_CLOSE_REGION-regionserver/c85114ed5096:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/B/4f3be9bfbee34ffaad735dfbcea9b838, entries=50, sequenceid=339, filesize=7.2 K 2024-11-22T03:41:47,084 DEBUG [RS_CLOSE_REGION-regionserver/c85114ed5096:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/.tmp/C/00c9f28186fb4ae3a7dfb5b58bee9d49 as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/C/00c9f28186fb4ae3a7dfb5b58bee9d49 2024-11-22T03:41:47,087 INFO [RS_CLOSE_REGION-regionserver/c85114ed5096:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/C/00c9f28186fb4ae3a7dfb5b58bee9d49, entries=50, sequenceid=339, filesize=7.2 K 2024-11-22T03:41:47,088 INFO [RS_CLOSE_REGION-regionserver/c85114ed5096:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HRegion(3140): Finished flush of dataSize ~13.42 KB/13740, heapSize ~35.86 KB/36720, currentSize=0 B/0 for 47b13a1c78111d05c87ac4dbad84cbd4 in 1661ms, sequenceid=339, compaction requested=true 2024-11-22T03:41:47,089 DEBUG [StoreCloser-TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/A/0451467100304ffdaa99be2cab971cb0, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/A/be8a456ced4d4c2da3441d8fa372a153, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/A/e3be43fa335c470b8184a779503e3b29, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/A/c2675a6a2c304fd98adb6a9713f9e799, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/A/b1584c8b0a4c49c2bfc79e77ad403d02, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/A/900cb9e957a0467b843ffdb809c5b0bd, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/A/6d72541868314cb9ac09d5f073f3615f, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/A/a97433bf816d40b1940a4ffb7706f0ed, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/A/20aea355efa74c579cd8979ab04d7d46, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/A/424bc496d28043e1a8b8cdb9b0c56b28, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/A/a1f31d0f8f1846d3b1b6ea15e35eeecb, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/A/511097a8881648ee82f0523a63983cfa, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/A/1420a2f7b6b841d7a8bf591779f57bba, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/A/73650cb54f9142bf8655a34da8b92b9a, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/A/6fcc9c37ffcb4a0da2008f32d9bfc6c4, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/A/16ac67e2790f4d9f860c5ef9171b7752, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/A/3ca0ad1a50e548088016069e85920764, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/A/4c80da1635604d1f803e0e1977bdcbcc, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/A/87bbeed4cbf049799a3bf7c69f85d06e, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/A/5ff59f4d55a14a669b5019a0332ea1e5, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/A/30d05496f27844f1ad2d5728d8058aca] to archive 2024-11-22T03:41:47,090 DEBUG [StoreCloser-TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-22T03:41:47,092 DEBUG [StoreCloser-TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/A/0451467100304ffdaa99be2cab971cb0 to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/A/0451467100304ffdaa99be2cab971cb0 2024-11-22T03:41:47,093 DEBUG [StoreCloser-TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/A/be8a456ced4d4c2da3441d8fa372a153 to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/A/be8a456ced4d4c2da3441d8fa372a153 2024-11-22T03:41:47,094 DEBUG [StoreCloser-TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/A/e3be43fa335c470b8184a779503e3b29 to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/A/e3be43fa335c470b8184a779503e3b29 2024-11-22T03:41:47,095 DEBUG [StoreCloser-TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/A/c2675a6a2c304fd98adb6a9713f9e799 to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/A/c2675a6a2c304fd98adb6a9713f9e799 2024-11-22T03:41:47,096 DEBUG [StoreCloser-TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/A/b1584c8b0a4c49c2bfc79e77ad403d02 to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/A/b1584c8b0a4c49c2bfc79e77ad403d02 2024-11-22T03:41:47,096 DEBUG [StoreCloser-TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/A/900cb9e957a0467b843ffdb809c5b0bd to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/A/900cb9e957a0467b843ffdb809c5b0bd 2024-11-22T03:41:47,097 DEBUG [StoreCloser-TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/A/6d72541868314cb9ac09d5f073f3615f to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/A/6d72541868314cb9ac09d5f073f3615f 2024-11-22T03:41:47,098 DEBUG [StoreCloser-TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/A/a97433bf816d40b1940a4ffb7706f0ed to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/A/a97433bf816d40b1940a4ffb7706f0ed 2024-11-22T03:41:47,099 DEBUG [StoreCloser-TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/A/20aea355efa74c579cd8979ab04d7d46 to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/A/20aea355efa74c579cd8979ab04d7d46 2024-11-22T03:41:47,100 DEBUG [StoreCloser-TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/A/424bc496d28043e1a8b8cdb9b0c56b28 to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/A/424bc496d28043e1a8b8cdb9b0c56b28 2024-11-22T03:41:47,100 DEBUG [StoreCloser-TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/A/a1f31d0f8f1846d3b1b6ea15e35eeecb to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/A/a1f31d0f8f1846d3b1b6ea15e35eeecb 2024-11-22T03:41:47,101 DEBUG [StoreCloser-TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/A/511097a8881648ee82f0523a63983cfa to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/A/511097a8881648ee82f0523a63983cfa 2024-11-22T03:41:47,102 DEBUG [StoreCloser-TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/A/1420a2f7b6b841d7a8bf591779f57bba to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/A/1420a2f7b6b841d7a8bf591779f57bba 2024-11-22T03:41:47,103 DEBUG [StoreCloser-TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/A/73650cb54f9142bf8655a34da8b92b9a to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/A/73650cb54f9142bf8655a34da8b92b9a 2024-11-22T03:41:47,104 DEBUG [StoreCloser-TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/A/6fcc9c37ffcb4a0da2008f32d9bfc6c4 to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/A/6fcc9c37ffcb4a0da2008f32d9bfc6c4 2024-11-22T03:41:47,104 DEBUG [StoreCloser-TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/A/16ac67e2790f4d9f860c5ef9171b7752 to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/A/16ac67e2790f4d9f860c5ef9171b7752 2024-11-22T03:41:47,105 DEBUG [StoreCloser-TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/A/3ca0ad1a50e548088016069e85920764 to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/A/3ca0ad1a50e548088016069e85920764 2024-11-22T03:41:47,106 DEBUG [StoreCloser-TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/A/4c80da1635604d1f803e0e1977bdcbcc to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/A/4c80da1635604d1f803e0e1977bdcbcc 2024-11-22T03:41:47,107 DEBUG [StoreCloser-TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/A/87bbeed4cbf049799a3bf7c69f85d06e to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/A/87bbeed4cbf049799a3bf7c69f85d06e 2024-11-22T03:41:47,107 DEBUG [StoreCloser-TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/A/5ff59f4d55a14a669b5019a0332ea1e5 to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/A/5ff59f4d55a14a669b5019a0332ea1e5 2024-11-22T03:41:47,108 DEBUG [StoreCloser-TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/A/30d05496f27844f1ad2d5728d8058aca to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/A/30d05496f27844f1ad2d5728d8058aca 2024-11-22T03:41:47,109 DEBUG [StoreCloser-TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/B/fcda2d06b5b1432c8d72e01dcf2d3980, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/B/66033de3255845928fb1a0fd8a15ba8a, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/B/7a56e285695344c1aad453081b77e8e7, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/B/e3ca809790034c49b0d925a373e412ff, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/B/de8f517c62e6458cb317bdbea9af310a, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/B/a669db6d679e41978776b81d8f176460, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/B/642df1c01ae243c6b2994695f3c7460d, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/B/d6907af7df39429da2da42cf852ed56b, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/B/259e5ab7e8cd4b9cbe981d4a044e47c3, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/B/9582e788caca4cc0ae6d9576a3abb322, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/B/f6213d5deb4c44feb5266a86d13f6cd2, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/B/5f375b04bd204edb87f27a5841598d06, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/B/4f834fc3ea1b40389e3f6368f244c2dc, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/B/742d540140d04a42b34418be4489f47f, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/B/8a6f7901af734d63a36cc32d5b681ca3, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/B/1257b2ad4ad5486386a698507848789d, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/B/b9f5be7ce49e4954a63880edbd83fc8a, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/B/e738aa7a0a4640f18c11a7d09571de4b, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/B/f7df388d64774bd3abfbd9ffa291111b, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/B/2b0501aae32140169b7fe29f9f5f06c7, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/B/c976218218f040939b42d53cd3946f8e] to archive 2024-11-22T03:41:47,110 DEBUG [StoreCloser-TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-22T03:41:47,111 DEBUG [StoreCloser-TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/B/fcda2d06b5b1432c8d72e01dcf2d3980 to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/B/fcda2d06b5b1432c8d72e01dcf2d3980 2024-11-22T03:41:47,112 DEBUG [StoreCloser-TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/B/66033de3255845928fb1a0fd8a15ba8a to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/B/66033de3255845928fb1a0fd8a15ba8a 2024-11-22T03:41:47,113 DEBUG [StoreCloser-TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/B/7a56e285695344c1aad453081b77e8e7 to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/B/7a56e285695344c1aad453081b77e8e7 2024-11-22T03:41:47,113 DEBUG [StoreCloser-TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/B/e3ca809790034c49b0d925a373e412ff to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/B/e3ca809790034c49b0d925a373e412ff 2024-11-22T03:41:47,114 DEBUG [StoreCloser-TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/B/de8f517c62e6458cb317bdbea9af310a to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/B/de8f517c62e6458cb317bdbea9af310a 2024-11-22T03:41:47,115 DEBUG [StoreCloser-TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/B/a669db6d679e41978776b81d8f176460 to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/B/a669db6d679e41978776b81d8f176460 2024-11-22T03:41:47,116 DEBUG [StoreCloser-TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/B/642df1c01ae243c6b2994695f3c7460d to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/B/642df1c01ae243c6b2994695f3c7460d 2024-11-22T03:41:47,116 DEBUG [StoreCloser-TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/B/d6907af7df39429da2da42cf852ed56b to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/B/d6907af7df39429da2da42cf852ed56b 2024-11-22T03:41:47,117 DEBUG [StoreCloser-TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/B/259e5ab7e8cd4b9cbe981d4a044e47c3 to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/B/259e5ab7e8cd4b9cbe981d4a044e47c3 2024-11-22T03:41:47,118 DEBUG [StoreCloser-TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/B/9582e788caca4cc0ae6d9576a3abb322 to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/B/9582e788caca4cc0ae6d9576a3abb322 2024-11-22T03:41:47,118 DEBUG [StoreCloser-TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/B/f6213d5deb4c44feb5266a86d13f6cd2 to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/B/f6213d5deb4c44feb5266a86d13f6cd2 2024-11-22T03:41:47,119 DEBUG [StoreCloser-TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/B/5f375b04bd204edb87f27a5841598d06 to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/B/5f375b04bd204edb87f27a5841598d06 2024-11-22T03:41:47,120 DEBUG [StoreCloser-TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/B/4f834fc3ea1b40389e3f6368f244c2dc to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/B/4f834fc3ea1b40389e3f6368f244c2dc 2024-11-22T03:41:47,121 DEBUG [StoreCloser-TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/B/742d540140d04a42b34418be4489f47f to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/B/742d540140d04a42b34418be4489f47f 2024-11-22T03:41:47,122 DEBUG [StoreCloser-TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/B/8a6f7901af734d63a36cc32d5b681ca3 to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/B/8a6f7901af734d63a36cc32d5b681ca3 2024-11-22T03:41:47,122 DEBUG [StoreCloser-TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/B/1257b2ad4ad5486386a698507848789d to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/B/1257b2ad4ad5486386a698507848789d 2024-11-22T03:41:47,123 DEBUG [StoreCloser-TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/B/b9f5be7ce49e4954a63880edbd83fc8a to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/B/b9f5be7ce49e4954a63880edbd83fc8a 2024-11-22T03:41:47,123 DEBUG [StoreCloser-TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/B/e738aa7a0a4640f18c11a7d09571de4b to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/B/e738aa7a0a4640f18c11a7d09571de4b 2024-11-22T03:41:47,124 DEBUG [StoreCloser-TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/B/f7df388d64774bd3abfbd9ffa291111b to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/B/f7df388d64774bd3abfbd9ffa291111b 2024-11-22T03:41:47,125 DEBUG [StoreCloser-TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/B/2b0501aae32140169b7fe29f9f5f06c7 to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/B/2b0501aae32140169b7fe29f9f5f06c7 2024-11-22T03:41:47,125 DEBUG [StoreCloser-TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/B/c976218218f040939b42d53cd3946f8e to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/B/c976218218f040939b42d53cd3946f8e 2024-11-22T03:41:47,127 DEBUG [StoreCloser-TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/C/ebb98f359c0c4eff9cb9b896c25d2092, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/C/d982949954404b3383facec71d14724d, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/C/fdaf9f11c8744da0abfd74f12f387052, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/C/9ac4244d179443ce867842ea26f08284, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/C/80ea1b6c83f94b2c96effac7620874c6, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/C/eb0db5ccd77d4cf3bfbaf64e3669fc33, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/C/5adf8b6acb714e57a81bbe590eb92096, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/C/36c79ec139df46868e02775e2147d904, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/C/26525a0fc0fb490d902d39bc1955bf76, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/C/f92470a0a7a748e98daa0cef10a47e3d, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/C/0949cc5df99b47edb00ea3acaae2efd5, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/C/e476a4c625644afc8f837f4f8ec07741, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/C/b3cc8352893e4c44bf65783ae90db563, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/C/19e2207df0f44c29bb29a3fca70a6791, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/C/29af4193c9384085bf01ff57b14e4402, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/C/516c3d5a186e4d159a071993f4ca8771, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/C/6251b861806c4b2f8d8db7cff1a7d194, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/C/9c06c50f053c4bffb458872d6aae4e3e, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/C/9227a1c151cb477ea8c5ff41fe46941e, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/C/e3f8522dbb7a4e7fb0973ec88b8b292a, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/C/79b0c67ae41d49388d8e367ffc08d5f5] to archive 2024-11-22T03:41:47,127 DEBUG [StoreCloser-TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-22T03:41:47,129 DEBUG [StoreCloser-TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/C/ebb98f359c0c4eff9cb9b896c25d2092 to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/C/ebb98f359c0c4eff9cb9b896c25d2092 2024-11-22T03:41:47,130 DEBUG [StoreCloser-TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/C/d982949954404b3383facec71d14724d to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/C/d982949954404b3383facec71d14724d 2024-11-22T03:41:47,130 DEBUG [StoreCloser-TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/C/fdaf9f11c8744da0abfd74f12f387052 to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/C/fdaf9f11c8744da0abfd74f12f387052 2024-11-22T03:41:47,131 DEBUG [StoreCloser-TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/C/9ac4244d179443ce867842ea26f08284 to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/C/9ac4244d179443ce867842ea26f08284 2024-11-22T03:41:47,132 DEBUG [StoreCloser-TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/C/80ea1b6c83f94b2c96effac7620874c6 to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/C/80ea1b6c83f94b2c96effac7620874c6 2024-11-22T03:41:47,133 DEBUG [StoreCloser-TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/C/eb0db5ccd77d4cf3bfbaf64e3669fc33 to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/C/eb0db5ccd77d4cf3bfbaf64e3669fc33 2024-11-22T03:41:47,134 DEBUG [StoreCloser-TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/C/5adf8b6acb714e57a81bbe590eb92096 to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/C/5adf8b6acb714e57a81bbe590eb92096 2024-11-22T03:41:47,134 DEBUG [StoreCloser-TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/C/36c79ec139df46868e02775e2147d904 to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/C/36c79ec139df46868e02775e2147d904 2024-11-22T03:41:47,135 DEBUG [StoreCloser-TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/C/26525a0fc0fb490d902d39bc1955bf76 to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/C/26525a0fc0fb490d902d39bc1955bf76 2024-11-22T03:41:47,136 DEBUG [StoreCloser-TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/C/f92470a0a7a748e98daa0cef10a47e3d to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/C/f92470a0a7a748e98daa0cef10a47e3d 2024-11-22T03:41:47,136 DEBUG [StoreCloser-TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/C/0949cc5df99b47edb00ea3acaae2efd5 to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/C/0949cc5df99b47edb00ea3acaae2efd5 2024-11-22T03:41:47,137 DEBUG [StoreCloser-TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/C/e476a4c625644afc8f837f4f8ec07741 to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/C/e476a4c625644afc8f837f4f8ec07741 2024-11-22T03:41:47,137 DEBUG [StoreCloser-TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/C/b3cc8352893e4c44bf65783ae90db563 to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/C/b3cc8352893e4c44bf65783ae90db563 2024-11-22T03:41:47,138 DEBUG [StoreCloser-TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/C/19e2207df0f44c29bb29a3fca70a6791 to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/C/19e2207df0f44c29bb29a3fca70a6791 2024-11-22T03:41:47,139 DEBUG [StoreCloser-TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/C/29af4193c9384085bf01ff57b14e4402 to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/C/29af4193c9384085bf01ff57b14e4402 2024-11-22T03:41:47,139 DEBUG [StoreCloser-TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/C/516c3d5a186e4d159a071993f4ca8771 to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/C/516c3d5a186e4d159a071993f4ca8771 2024-11-22T03:41:47,140 DEBUG [StoreCloser-TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/C/6251b861806c4b2f8d8db7cff1a7d194 to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/C/6251b861806c4b2f8d8db7cff1a7d194 2024-11-22T03:41:47,140 DEBUG [StoreCloser-TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/C/9c06c50f053c4bffb458872d6aae4e3e to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/C/9c06c50f053c4bffb458872d6aae4e3e 2024-11-22T03:41:47,141 DEBUG [StoreCloser-TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/C/9227a1c151cb477ea8c5ff41fe46941e to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/C/9227a1c151cb477ea8c5ff41fe46941e 2024-11-22T03:41:47,142 DEBUG [StoreCloser-TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/C/e3f8522dbb7a4e7fb0973ec88b8b292a to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/C/e3f8522dbb7a4e7fb0973ec88b8b292a 2024-11-22T03:41:47,142 DEBUG [StoreCloser-TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/C/79b0c67ae41d49388d8e367ffc08d5f5 to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/C/79b0c67ae41d49388d8e367ffc08d5f5 2024-11-22T03:41:47,145 DEBUG [RS_CLOSE_REGION-regionserver/c85114ed5096:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/recovered.edits/342.seqid, newMaxSeqId=342, maxSeqId=4 2024-11-22T03:41:47,146 INFO [RS_CLOSE_REGION-regionserver/c85114ed5096:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4. 2024-11-22T03:41:47,146 DEBUG [RS_CLOSE_REGION-regionserver/c85114ed5096:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HRegion(1676): Region close journal for 47b13a1c78111d05c87ac4dbad84cbd4: Waiting for close lock at 1732246904453Running coprocessor pre-close hooks at 1732246904453Disabling compacts and flushes for region at 1732246904453Disabling writes for close at 1732246905427 (+974 ms)Obtaining lock to block concurrent updates at 1732246905427Preparing flush snapshotting stores in 47b13a1c78111d05c87ac4dbad84cbd4 at 1732246905427Finished memstore snapshotting TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4., syncing WAL and waiting on mvcc, flushsize=dataSize=13740, getHeapSize=36720, getOffHeapSize=0, getCellsCount=300 at 1732246905428 (+1 ms)Flushing stores of TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4. at 1732246905428Flushing 47b13a1c78111d05c87ac4dbad84cbd4/A: creating writer at 1732246905428Flushing mob file 47b13a1c78111d05c87ac4dbad84cbd4/A: appending metadata at 1732246905431 (+3 ms)Flushing mob file 47b13a1c78111d05c87ac4dbad84cbd4/A: closing flushed file at 1732246905432 (+1 ms)Flushing 47b13a1c78111d05c87ac4dbad84cbd4/A: appending metadata at 1732246905842 (+410 ms)Flushing 47b13a1c78111d05c87ac4dbad84cbd4/A: closing flushed file at 1732246905842Flushing 47b13a1c78111d05c87ac4dbad84cbd4/B: creating writer at 1732246906249 (+407 ms)Flushing 47b13a1c78111d05c87ac4dbad84cbd4/B: appending metadata at 1732246906250 (+1 ms)Flushing 47b13a1c78111d05c87ac4dbad84cbd4/B: closing flushed file at 1732246906250Flushing 47b13a1c78111d05c87ac4dbad84cbd4/C: creating writer at 1732246906657 (+407 ms)Flushing 47b13a1c78111d05c87ac4dbad84cbd4/C: appending metadata at 1732246906659 (+2 ms)Flushing 47b13a1c78111d05c87ac4dbad84cbd4/C: closing flushed file at 1732246906659Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@8527c92: reopening flushed file at 1732246907071 (+412 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6dabf75d: reopening flushed file at 1732246907078 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5216f907: reopening flushed file at 1732246907083 (+5 ms)Finished flush of dataSize ~13.42 KB/13740, heapSize ~35.86 KB/36720, currentSize=0 B/0 for 47b13a1c78111d05c87ac4dbad84cbd4 in 1661ms, sequenceid=339, compaction requested=true at 1732246907088 (+5 ms)Writing region close event to WAL at 1732246907143 (+55 ms)Running coprocessor post-close hooks at 1732246907146 (+3 ms)Closed at 1732246907146 2024-11-22T03:41:47,147 INFO [RS_CLOSE_REGION-regionserver/c85114ed5096:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] handler.UnassignRegionHandler(157): Closed 47b13a1c78111d05c87ac4dbad84cbd4 2024-11-22T03:41:47,148 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=111 updating hbase:meta row=47b13a1c78111d05c87ac4dbad84cbd4, regionState=CLOSED 2024-11-22T03:41:47,149 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=112, ppid=111, state=RUNNABLE, hasLock=false; CloseRegionProcedure 47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 because future has completed 2024-11-22T03:41:47,150 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=112, resume processing ppid=111 2024-11-22T03:41:47,150 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=112, ppid=111, state=SUCCESS, hasLock=false; CloseRegionProcedure 47b13a1c78111d05c87ac4dbad84cbd4, server=c85114ed5096,42545,1732246786482 in 2.8480 sec 2024-11-22T03:41:47,152 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=111, resume processing ppid=110 2024-11-22T03:41:47,152 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=111, ppid=110, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=47b13a1c78111d05c87ac4dbad84cbd4, UNASSIGN in 2.8520 sec 2024-11-22T03:41:47,153 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=110, resume processing ppid=109 2024-11-22T03:41:47,153 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=110, ppid=109, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 2.8550 sec 2024-11-22T03:41:47,154 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732246907154"}]},"ts":"1732246907154"} 2024-11-22T03:41:47,155 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-22T03:41:47,155 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-22T03:41:47,156 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=109, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 2.8640 sec 2024-11-22T03:41:48,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-22T03:41:48,459 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-22T03:41:48,459 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42627 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-22T03:41:48,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42627 {}] procedure2.ProcedureExecutor(1139): Stored pid=113, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-22T03:41:48,461 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=113, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-22T03:41:48,462 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=113, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-22T03:41:48,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-22T03:41:48,464 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4 2024-11-22T03:41:48,467 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/A, FileablePath, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/B, FileablePath, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/C, FileablePath, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/recovered.edits] 2024-11-22T03:41:48,469 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/A/2439bbfd5ad9412d80dbec229fd894bb to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/A/2439bbfd5ad9412d80dbec229fd894bb 2024-11-22T03:41:48,470 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/A/3899fddd1e9c4d2e9c1a54ea1158e121 to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/A/3899fddd1e9c4d2e9c1a54ea1158e121 2024-11-22T03:41:48,470 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/A/5088a2ac058f4963b79b71d9e91654b3 to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/A/5088a2ac058f4963b79b71d9e91654b3 2024-11-22T03:41:48,471 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/A/f8af0228d5164e5c81870244e1412d40 to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/A/f8af0228d5164e5c81870244e1412d40 2024-11-22T03:41:48,473 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/B/2f1e2ed457ab4c44aad5c41664f4e5ea to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/B/2f1e2ed457ab4c44aad5c41664f4e5ea 2024-11-22T03:41:48,474 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/B/4f3be9bfbee34ffaad735dfbcea9b838 to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/B/4f3be9bfbee34ffaad735dfbcea9b838 2024-11-22T03:41:48,474 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/B/943c0c5eb2b94c4d9012fd5cb9f3a730 to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/B/943c0c5eb2b94c4d9012fd5cb9f3a730 2024-11-22T03:41:48,475 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/B/d25e75af22ed4441915ae43f3ae7885a to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/B/d25e75af22ed4441915ae43f3ae7885a 2024-11-22T03:41:48,476 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/C/00c9f28186fb4ae3a7dfb5b58bee9d49 to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/C/00c9f28186fb4ae3a7dfb5b58bee9d49 2024-11-22T03:41:48,477 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/C/2046c80283f84ba9afb12e68d1e07d3f to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/C/2046c80283f84ba9afb12e68d1e07d3f 2024-11-22T03:41:48,478 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/C/7766eef7d3d248f888df73db4248925a to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/C/7766eef7d3d248f888df73db4248925a 2024-11-22T03:41:48,478 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/C/d43365d32aac4cd0801ec0753618022e to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/C/d43365d32aac4cd0801ec0753618022e 2024-11-22T03:41:48,480 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/recovered.edits/342.seqid to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4/recovered.edits/342.seqid 2024-11-22T03:41:48,480 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/47b13a1c78111d05c87ac4dbad84cbd4 2024-11-22T03:41:48,480 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-22T03:41:48,481 DEBUG [PEWorker-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-22T03:41:48,481 DEBUG [PEWorker-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-22T03:41:48,483 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411220617bd7f18624fda87545997072dea34_47b13a1c78111d05c87ac4dbad84cbd4 to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411220617bd7f18624fda87545997072dea34_47b13a1c78111d05c87ac4dbad84cbd4 2024-11-22T03:41:48,484 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112211d7ca2c98984f5e86e35a34f452207e_47b13a1c78111d05c87ac4dbad84cbd4 to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112211d7ca2c98984f5e86e35a34f452207e_47b13a1c78111d05c87ac4dbad84cbd4 2024-11-22T03:41:48,485 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112225c32beb1db649f3afd1de8cf2090e12_47b13a1c78111d05c87ac4dbad84cbd4 to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112225c32beb1db649f3afd1de8cf2090e12_47b13a1c78111d05c87ac4dbad84cbd4 2024-11-22T03:41:48,485 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112244dc6e625f2845a5b63a15a7dfbe2ea5_47b13a1c78111d05c87ac4dbad84cbd4 to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112244dc6e625f2845a5b63a15a7dfbe2ea5_47b13a1c78111d05c87ac4dbad84cbd4 2024-11-22T03:41:48,486 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411226165be01e57a4cd7a6e232f6c41ea137_47b13a1c78111d05c87ac4dbad84cbd4 to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411226165be01e57a4cd7a6e232f6c41ea137_47b13a1c78111d05c87ac4dbad84cbd4 2024-11-22T03:41:48,487 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112289c0741aa1464e9b9fa277e16cd046ef_47b13a1c78111d05c87ac4dbad84cbd4 to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112289c0741aa1464e9b9fa277e16cd046ef_47b13a1c78111d05c87ac4dbad84cbd4 2024-11-22T03:41:48,487 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411228c3bcea1cc3641bdaddb6b2f10b35260_47b13a1c78111d05c87ac4dbad84cbd4 to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411228c3bcea1cc3641bdaddb6b2f10b35260_47b13a1c78111d05c87ac4dbad84cbd4 2024-11-22T03:41:48,488 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241122b6899f47bd6344dc9490b74d7a5b4599_47b13a1c78111d05c87ac4dbad84cbd4 to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241122b6899f47bd6344dc9490b74d7a5b4599_47b13a1c78111d05c87ac4dbad84cbd4 2024-11-22T03:41:48,489 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241122b9a0c9e9dea64747b2f98f87508db16b_47b13a1c78111d05c87ac4dbad84cbd4 to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241122b9a0c9e9dea64747b2f98f87508db16b_47b13a1c78111d05c87ac4dbad84cbd4 2024-11-22T03:41:48,489 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241122bd08256b4b7c4e6ea0ee8a26f7a94751_47b13a1c78111d05c87ac4dbad84cbd4 to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241122bd08256b4b7c4e6ea0ee8a26f7a94751_47b13a1c78111d05c87ac4dbad84cbd4 2024-11-22T03:41:48,490 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241122c91c444343b245159aba4269f75696f2_47b13a1c78111d05c87ac4dbad84cbd4 to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241122c91c444343b245159aba4269f75696f2_47b13a1c78111d05c87ac4dbad84cbd4 2024-11-22T03:41:48,491 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241122eb6f5239900345108db8d804dd4895a7_47b13a1c78111d05c87ac4dbad84cbd4 to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241122eb6f5239900345108db8d804dd4895a7_47b13a1c78111d05c87ac4dbad84cbd4 2024-11-22T03:41:48,491 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241122f157dbfd6b8243a7bd204bf9a4a915cb_47b13a1c78111d05c87ac4dbad84cbd4 to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241122f157dbfd6b8243a7bd204bf9a4a915cb_47b13a1c78111d05c87ac4dbad84cbd4 2024-11-22T03:41:48,492 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241122f20d3251941948dab91f6c2076aca01c_47b13a1c78111d05c87ac4dbad84cbd4 to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241122f20d3251941948dab91f6c2076aca01c_47b13a1c78111d05c87ac4dbad84cbd4 2024-11-22T03:41:48,493 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241122f2d5c73d76934823abd42559eaf16da2_47b13a1c78111d05c87ac4dbad84cbd4 to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241122f2d5c73d76934823abd42559eaf16da2_47b13a1c78111d05c87ac4dbad84cbd4 2024-11-22T03:41:48,494 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241122f90652dad6cb43398e76618c1f02763e_47b13a1c78111d05c87ac4dbad84cbd4 to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241122f90652dad6cb43398e76618c1f02763e_47b13a1c78111d05c87ac4dbad84cbd4 2024-11-22T03:41:48,494 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241122fed56234794943c289e4d61df78c549c_47b13a1c78111d05c87ac4dbad84cbd4 to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241122fed56234794943c289e4d61df78c549c_47b13a1c78111d05c87ac4dbad84cbd4 2024-11-22T03:41:48,495 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241122ffb1a02109bc4ba583378aa83244dcf1_47b13a1c78111d05c87ac4dbad84cbd4 to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241122ffb1a02109bc4ba583378aa83244dcf1_47b13a1c78111d05c87ac4dbad84cbd4 2024-11-22T03:41:48,496 DEBUG [PEWorker-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-22T03:41:48,497 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=113, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-22T03:41:48,498 WARN [PEWorker-1 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-22T03:41:48,500 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-22T03:41:48,500 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=113, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-22T03:41:48,500 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-22T03:41:48,501 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732246908500"}]},"ts":"9223372036854775807"} 2024-11-22T03:41:48,502 INFO [PEWorker-1 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-22T03:41:48,502 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 47b13a1c78111d05c87ac4dbad84cbd4, NAME => 'TestAcidGuarantees,,1732246880600.47b13a1c78111d05c87ac4dbad84cbd4.', STARTKEY => '', ENDKEY => ''}] 2024-11-22T03:41:48,502 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-22T03:41:48,502 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732246908502"}]},"ts":"9223372036854775807"} 2024-11-22T03:41:48,503 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-22T03:41:48,504 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(133): Finished pid=113, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-22T03:41:48,505 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=113, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 45 msec 2024-11-22T03:41:48,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-22T03:41:48,569 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-22T03:41:48,569 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-22T03:41:48,576 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testMobScanAtomicity Thread=241 (was 242), OpenFileDescriptor=461 (was 459) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=381 (was 358) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=3234 (was 2992) - AvailableMemoryMB LEAK? - 2024-11-22T03:41:48,583 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testScanAtomicity Thread=241, OpenFileDescriptor=461, MaxFileDescriptor=1048576, SystemLoadAverage=381, ProcessCount=11, AvailableMemoryMB=3232 2024-11-22T03:41:48,584 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42627 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-22T03:41:48,584 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42627 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-22T03:41:48,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42627 {}] procedure2.ProcedureExecutor(1139): Stored pid=114, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-22T03:41:48,586 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=114, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-22T03:41:48,586 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:48,586 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42627 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 114 2024-11-22T03:41:48,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=114 2024-11-22T03:41:48,587 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=114, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-22T03:41:48,591 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742218_1394 (size=959) 2024-11-22T03:41:48,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=114 2024-11-22T03:41:48,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=114 2024-11-22T03:41:48,993 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 4a7ce0cfb17f4dcc3dd24b2156963795, NAME => 'TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a 2024-11-22T03:41:48,998 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742219_1395 (size=53) 2024-11-22T03:41:49,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=114 2024-11-22T03:41:49,399 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-22T03:41:49,399 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 4a7ce0cfb17f4dcc3dd24b2156963795, disabling compactions & flushes 2024-11-22T03:41:49,399 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795. 2024-11-22T03:41:49,399 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795. 2024-11-22T03:41:49,399 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795. after waiting 0 ms 2024-11-22T03:41:49,399 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795. 2024-11-22T03:41:49,399 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795. 2024-11-22T03:41:49,399 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 4a7ce0cfb17f4dcc3dd24b2156963795: Waiting for close lock at 1732246909399Disabling compacts and flushes for region at 1732246909399Disabling writes for close at 1732246909399Writing region close event to WAL at 1732246909399Closed at 1732246909399 2024-11-22T03:41:49,400 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=114, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-22T03:41:49,400 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732246909400"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732246909400"}]},"ts":"1732246909400"} 2024-11-22T03:41:49,401 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-22T03:41:49,402 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=114, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-22T03:41:49,402 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732246909402"}]},"ts":"1732246909402"} 2024-11-22T03:41:49,403 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-22T03:41:49,403 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=115, ppid=114, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=4a7ce0cfb17f4dcc3dd24b2156963795, ASSIGN}] 2024-11-22T03:41:49,404 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=115, ppid=114, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=4a7ce0cfb17f4dcc3dd24b2156963795, ASSIGN 2024-11-22T03:41:49,405 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=115, ppid=114, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=4a7ce0cfb17f4dcc3dd24b2156963795, ASSIGN; state=OFFLINE, location=c85114ed5096,42545,1732246786482; forceNewPlan=false, retain=false 2024-11-22T03:41:49,410 DEBUG [regionserver/c85114ed5096:0.Chore.1 {}] throttle.PressureAwareCompactionThroughputController(103): CompactionPressure is 0.0, tune throughput to 50.00 MB/second 2024-11-22T03:41:49,556 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=115 updating hbase:meta row=4a7ce0cfb17f4dcc3dd24b2156963795, regionState=OPENING, regionLocation=c85114ed5096,42545,1732246786482 2024-11-22T03:41:49,557 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=115, ppid=114, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=4a7ce0cfb17f4dcc3dd24b2156963795, ASSIGN because future has completed 2024-11-22T03:41:49,557 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=116, ppid=115, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482}] 2024-11-22T03:41:49,715 INFO [RS_OPEN_REGION-regionserver/c85114ed5096:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795. 2024-11-22T03:41:49,715 DEBUG [RS_OPEN_REGION-regionserver/c85114ed5096:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegion(7752): Opening region: {ENCODED => 4a7ce0cfb17f4dcc3dd24b2156963795, NAME => 'TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795.', STARTKEY => '', ENDKEY => ''} 2024-11-22T03:41:49,717 DEBUG [RS_OPEN_REGION-regionserver/c85114ed5096:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 4a7ce0cfb17f4dcc3dd24b2156963795 2024-11-22T03:41:49,717 DEBUG [RS_OPEN_REGION-regionserver/c85114ed5096:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-22T03:41:49,717 DEBUG [RS_OPEN_REGION-regionserver/c85114ed5096:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegion(7794): checking encryption for 4a7ce0cfb17f4dcc3dd24b2156963795 2024-11-22T03:41:49,717 DEBUG [RS_OPEN_REGION-regionserver/c85114ed5096:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegion(7797): checking classloading for 4a7ce0cfb17f4dcc3dd24b2156963795 2024-11-22T03:41:49,719 INFO [StoreOpener-4a7ce0cfb17f4dcc3dd24b2156963795-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 4a7ce0cfb17f4dcc3dd24b2156963795 2024-11-22T03:41:49,721 INFO [StoreOpener-4a7ce0cfb17f4dcc3dd24b2156963795-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4a7ce0cfb17f4dcc3dd24b2156963795 columnFamilyName A 2024-11-22T03:41:49,721 DEBUG [StoreOpener-4a7ce0cfb17f4dcc3dd24b2156963795-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:49,721 INFO [StoreOpener-4a7ce0cfb17f4dcc3dd24b2156963795-1 {}] regionserver.HStore(327): Store=4a7ce0cfb17f4dcc3dd24b2156963795/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-22T03:41:49,722 INFO [StoreOpener-4a7ce0cfb17f4dcc3dd24b2156963795-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 4a7ce0cfb17f4dcc3dd24b2156963795 2024-11-22T03:41:49,723 INFO [StoreOpener-4a7ce0cfb17f4dcc3dd24b2156963795-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4a7ce0cfb17f4dcc3dd24b2156963795 columnFamilyName B 2024-11-22T03:41:49,723 DEBUG [StoreOpener-4a7ce0cfb17f4dcc3dd24b2156963795-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:49,724 INFO [StoreOpener-4a7ce0cfb17f4dcc3dd24b2156963795-1 {}] regionserver.HStore(327): Store=4a7ce0cfb17f4dcc3dd24b2156963795/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-22T03:41:49,724 INFO [StoreOpener-4a7ce0cfb17f4dcc3dd24b2156963795-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 4a7ce0cfb17f4dcc3dd24b2156963795 2024-11-22T03:41:49,726 INFO [StoreOpener-4a7ce0cfb17f4dcc3dd24b2156963795-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4a7ce0cfb17f4dcc3dd24b2156963795 columnFamilyName C 2024-11-22T03:41:49,726 DEBUG [StoreOpener-4a7ce0cfb17f4dcc3dd24b2156963795-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:41:49,727 INFO [StoreOpener-4a7ce0cfb17f4dcc3dd24b2156963795-1 {}] regionserver.HStore(327): Store=4a7ce0cfb17f4dcc3dd24b2156963795/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-22T03:41:49,727 DEBUG [RS_OPEN_REGION-regionserver/c85114ed5096:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegion(1038): replaying wal for 4a7ce0cfb17f4dcc3dd24b2156963795 2024-11-22T03:41:49,728 DEBUG [RS_OPEN_REGION-regionserver/c85114ed5096:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795 2024-11-22T03:41:49,729 DEBUG [RS_OPEN_REGION-regionserver/c85114ed5096:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795 2024-11-22T03:41:49,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=114 2024-11-22T03:41:49,731 DEBUG [RS_OPEN_REGION-regionserver/c85114ed5096:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegion(1048): stopping wal replay for 4a7ce0cfb17f4dcc3dd24b2156963795 2024-11-22T03:41:49,731 DEBUG [RS_OPEN_REGION-regionserver/c85114ed5096:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegion(1060): Cleaning up temporary data for 4a7ce0cfb17f4dcc3dd24b2156963795 2024-11-22T03:41:49,731 DEBUG [RS_OPEN_REGION-regionserver/c85114ed5096:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-22T03:41:49,734 DEBUG [RS_OPEN_REGION-regionserver/c85114ed5096:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegion(1093): writing seq id for 4a7ce0cfb17f4dcc3dd24b2156963795 2024-11-22T03:41:49,736 DEBUG [RS_OPEN_REGION-regionserver/c85114ed5096:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-22T03:41:49,737 INFO [RS_OPEN_REGION-regionserver/c85114ed5096:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegion(1114): Opened 4a7ce0cfb17f4dcc3dd24b2156963795; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67052019, jitterRate=-8.4705650806427E-4}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-22T03:41:49,737 DEBUG [RS_OPEN_REGION-regionserver/c85114ed5096:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 4a7ce0cfb17f4dcc3dd24b2156963795 2024-11-22T03:41:49,738 DEBUG [RS_OPEN_REGION-regionserver/c85114ed5096:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegion(1006): Region open journal for 4a7ce0cfb17f4dcc3dd24b2156963795: Running coprocessor pre-open hook at 1732246909717Writing region info on filesystem at 1732246909717Initializing all the Stores at 1732246909718 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732246909718Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732246909719 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732246909719Cleaning up temporary data from old regions at 1732246909731 (+12 ms)Running coprocessor post-open hooks at 1732246909737 (+6 ms)Region opened successfully at 1732246909738 (+1 ms) 2024-11-22T03:41:49,739 INFO [RS_OPEN_REGION-regionserver/c85114ed5096:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., pid=116, masterSystemTime=1732246909709 2024-11-22T03:41:49,742 DEBUG [RS_OPEN_REGION-regionserver/c85114ed5096:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795. 2024-11-22T03:41:49,742 INFO [RS_OPEN_REGION-regionserver/c85114ed5096:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795. 2024-11-22T03:41:49,743 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=115 updating hbase:meta row=4a7ce0cfb17f4dcc3dd24b2156963795, regionState=OPEN, openSeqNum=2, regionLocation=c85114ed5096,42545,1732246786482 2024-11-22T03:41:49,744 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=116, ppid=115, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 because future has completed 2024-11-22T03:41:49,747 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=116, resume processing ppid=115 2024-11-22T03:41:49,747 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=116, ppid=115, state=SUCCESS, hasLock=false; OpenRegionProcedure 4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 in 188 msec 2024-11-22T03:41:49,750 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=115, resume processing ppid=114 2024-11-22T03:41:49,750 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=115, ppid=114, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=4a7ce0cfb17f4dcc3dd24b2156963795, ASSIGN in 344 msec 2024-11-22T03:41:49,751 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=114, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-22T03:41:49,751 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732246909751"}]},"ts":"1732246909751"} 2024-11-22T03:41:49,752 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-22T03:41:49,753 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=114, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-22T03:41:49,755 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=114, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1690 sec 2024-11-22T03:41:50,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=114 2024-11-22T03:41:50,739 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-22T03:41:50,740 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@72240ede, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T03:41:50,740 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c85114ed5096,42627,-1 for getting cluster id 2024-11-22T03:41:50,740 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-22T03:41:50,741 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '67035b7f-c649-44a8-9db7-17adff49cedb' 2024-11-22T03:41:50,741 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-22T03:41:50,741 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "67035b7f-c649-44a8-9db7-17adff49cedb" 2024-11-22T03:41:50,742 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7fac5809, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T03:41:50,742 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c85114ed5096,42627,-1] 2024-11-22T03:41:50,742 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-22T03:41:50,742 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T03:41:50,743 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36020, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-22T03:41:50,743 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@70c8ae9f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T03:41:50,743 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-22T03:41:50,744 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c85114ed5096,42545,1732246786482, seqNum=-1] 2024-11-22T03:41:50,744 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-22T03:41:50,745 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40918, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-22T03:41:50,746 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-22T03:41:50,746 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncConnectionImpl(321): The fetched master address is c85114ed5096,42627,1732246785796 2024-11-22T03:41:50,746 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@2fb3b6d0 2024-11-22T03:41:50,746 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-22T03:41:50,747 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36024, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-22T03:41:50,749 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@503e1f3c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T03:41:50,749 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c85114ed5096,42627,-1 for getting cluster id 2024-11-22T03:41:50,749 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-22T03:41:50,749 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '67035b7f-c649-44a8-9db7-17adff49cedb' 2024-11-22T03:41:50,750 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-22T03:41:50,750 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "67035b7f-c649-44a8-9db7-17adff49cedb" 2024-11-22T03:41:50,750 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@535df2f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T03:41:50,750 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c85114ed5096,42627,-1] 2024-11-22T03:41:50,750 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-22T03:41:50,750 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T03:41:50,751 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36054, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-22T03:41:50,751 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@24897023, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T03:41:50,752 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@205b33d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T03:41:50,752 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c85114ed5096,42627,-1 for getting cluster id 2024-11-22T03:41:50,752 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-22T03:41:50,753 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '67035b7f-c649-44a8-9db7-17adff49cedb' 2024-11-22T03:41:50,753 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-22T03:41:50,753 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "67035b7f-c649-44a8-9db7-17adff49cedb" 2024-11-22T03:41:50,753 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@41311869, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T03:41:50,753 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c85114ed5096,42627,-1] 2024-11-22T03:41:50,753 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-22T03:41:50,753 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T03:41:50,754 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36080, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-22T03:41:50,754 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@59991c42, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T03:41:50,755 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7e100fd5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T03:41:50,755 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c85114ed5096,42627,-1 for getting cluster id 2024-11-22T03:41:50,755 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-22T03:41:50,755 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '67035b7f-c649-44a8-9db7-17adff49cedb' 2024-11-22T03:41:50,755 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-22T03:41:50,756 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "67035b7f-c649-44a8-9db7-17adff49cedb" 2024-11-22T03:41:50,756 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@a280275, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T03:41:50,756 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c85114ed5096,42627,-1] 2024-11-22T03:41:50,756 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-22T03:41:50,756 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T03:41:50,756 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36102, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-22T03:41:50,757 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5fa64f3f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T03:41:50,757 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@58e40afb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T03:41:50,757 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c85114ed5096,42627,-1 for getting cluster id 2024-11-22T03:41:50,758 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-22T03:41:50,758 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '67035b7f-c649-44a8-9db7-17adff49cedb' 2024-11-22T03:41:50,758 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-22T03:41:50,758 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "67035b7f-c649-44a8-9db7-17adff49cedb" 2024-11-22T03:41:50,758 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@20605f14, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T03:41:50,758 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c85114ed5096,42627,-1] 2024-11-22T03:41:50,758 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-22T03:41:50,759 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T03:41:50,759 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36126, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-22T03:41:50,759 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4351b69e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T03:41:50,760 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1b9f4da0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T03:41:50,760 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c85114ed5096,42627,-1 for getting cluster id 2024-11-22T03:41:50,760 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-22T03:41:50,761 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '67035b7f-c649-44a8-9db7-17adff49cedb' 2024-11-22T03:41:50,761 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-22T03:41:50,761 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "67035b7f-c649-44a8-9db7-17adff49cedb" 2024-11-22T03:41:50,761 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@361431aa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T03:41:50,761 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c85114ed5096,42627,-1] 2024-11-22T03:41:50,761 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-22T03:41:50,761 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T03:41:50,762 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36156, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-22T03:41:50,762 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@600ad8e7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T03:41:50,762 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3e4cbf6b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T03:41:50,762 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c85114ed5096,42627,-1 for getting cluster id 2024-11-22T03:41:50,763 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-22T03:41:50,763 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '67035b7f-c649-44a8-9db7-17adff49cedb' 2024-11-22T03:41:50,763 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-22T03:41:50,763 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "67035b7f-c649-44a8-9db7-17adff49cedb" 2024-11-22T03:41:50,763 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2a06ec3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T03:41:50,763 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c85114ed5096,42627,-1] 2024-11-22T03:41:50,763 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-22T03:41:50,764 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T03:41:50,764 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36180, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-22T03:41:50,765 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3ab10361, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T03:41:50,765 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4c83334c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T03:41:50,765 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c85114ed5096,42627,-1 for getting cluster id 2024-11-22T03:41:50,765 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-22T03:41:50,766 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '67035b7f-c649-44a8-9db7-17adff49cedb' 2024-11-22T03:41:50,766 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-22T03:41:50,766 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "67035b7f-c649-44a8-9db7-17adff49cedb" 2024-11-22T03:41:50,766 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@265b04a0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T03:41:50,766 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c85114ed5096,42627,-1] 2024-11-22T03:41:50,766 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-22T03:41:50,767 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T03:41:50,767 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36202, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-22T03:41:50,768 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5e5a33a7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T03:41:50,768 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3532b5c1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T03:41:50,768 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c85114ed5096,42627,-1 for getting cluster id 2024-11-22T03:41:50,769 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-22T03:41:50,769 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '67035b7f-c649-44a8-9db7-17adff49cedb' 2024-11-22T03:41:50,769 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-22T03:41:50,769 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "67035b7f-c649-44a8-9db7-17adff49cedb" 2024-11-22T03:41:50,770 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@63d4485c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T03:41:50,770 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c85114ed5096,42627,-1] 2024-11-22T03:41:50,770 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-22T03:41:50,770 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T03:41:50,771 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36224, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-22T03:41:50,771 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2f1724fb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T03:41:50,773 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3088fab0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T03:41:50,773 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c85114ed5096,42627,-1 for getting cluster id 2024-11-22T03:41:50,773 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-22T03:41:50,774 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '67035b7f-c649-44a8-9db7-17adff49cedb' 2024-11-22T03:41:50,774 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-22T03:41:50,774 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "67035b7f-c649-44a8-9db7-17adff49cedb" 2024-11-22T03:41:50,774 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@62d397af, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T03:41:50,774 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c85114ed5096,42627,-1] 2024-11-22T03:41:50,774 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-22T03:41:50,774 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T03:41:50,775 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36242, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-22T03:41:50,775 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7a41f9ec, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T03:41:50,776 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@663bb5d5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T03:41:50,776 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c85114ed5096,42627,-1 for getting cluster id 2024-11-22T03:41:50,776 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-22T03:41:50,777 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '67035b7f-c649-44a8-9db7-17adff49cedb' 2024-11-22T03:41:50,777 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-22T03:41:50,777 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "67035b7f-c649-44a8-9db7-17adff49cedb" 2024-11-22T03:41:50,777 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32482c87, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T03:41:50,777 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c85114ed5096,42627,-1] 2024-11-22T03:41:50,777 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-22T03:41:50,777 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T03:41:50,778 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36250, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-22T03:41:50,778 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@13ff3778, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T03:41:50,781 DEBUG [Thread-1816 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-22T03:41:50,781 DEBUG [Thread-1801 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-22T03:41:50,781 DEBUG [Thread-1812 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-22T03:41:50,781 DEBUG [Thread-1803 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-22T03:41:50,781 DEBUG [Thread-1805 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-22T03:41:50,782 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c85114ed5096,42545,1732246786482, seqNum=-1] 2024-11-22T03:41:50,782 DEBUG [Thread-1805 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c85114ed5096,42545,1732246786482, seqNum=-1] 2024-11-22T03:41:50,782 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42627 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-22T03:41:50,782 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-22T03:41:50,782 DEBUG [Thread-1805 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-22T03:41:50,783 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c85114ed5096,42545,1732246786482, seqNum=-1] 2024-11-22T03:41:50,783 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-22T03:41:50,783 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c85114ed5096,42545,1732246786482, seqNum=-1] 2024-11-22T03:41:50,784 DEBUG [Thread-1818 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-22T03:41:50,784 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-22T03:41:50,784 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c85114ed5096,42545,1732246786482, seqNum=-1] 2024-11-22T03:41:50,784 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40926, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-22T03:41:50,784 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40938, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-22T03:41:50,784 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-22T03:41:50,784 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40954, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-22T03:41:50,784 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c85114ed5096,42545,1732246786482, seqNum=-1] 2024-11-22T03:41:50,785 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-22T03:41:50,785 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40962, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-22T03:41:50,785 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40966, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-22T03:41:50,785 DEBUG [Thread-1807 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-22T03:41:50,786 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2] 2024-11-22T03:41:50,786 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c85114ed5096,42545,1732246786482, seqNum=-1] 2024-11-22T03:41:50,786 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-22T03:41:50,787 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40974, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-22T03:41:50,787 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2] 2024-11-22T03:41:50,788 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40990, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-22T03:41:50,788 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2] 2024-11-22T03:41:50,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42627 {}] procedure2.ProcedureExecutor(1139): Stored pid=117, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees 2024-11-22T03:41:50,789 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2] 2024-11-22T03:41:50,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-22T03:41:50,790 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2] 2024-11-22T03:41:50,790 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2] 2024-11-22T03:41:50,791 DEBUG [Thread-1814 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-22T03:41:50,791 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2] 2024-11-22T03:41:50,791 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=117, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-22T03:41:50,792 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c85114ed5096,42545,1732246786482, seqNum=-1] 2024-11-22T03:41:50,792 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=117, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-22T03:41:50,792 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=118, ppid=117, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-22T03:41:50,792 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-22T03:41:50,793 DEBUG [Thread-1809 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-22T03:41:50,793 DEBUG [Thread-1820 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-22T03:41:50,794 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40998, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-22T03:41:50,794 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c85114ed5096,42545,1732246786482, seqNum=-1] 2024-11-22T03:41:50,794 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-22T03:41:50,796 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2] 2024-11-22T03:41:50,796 DEBUG [Thread-1809 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c85114ed5096,42545,1732246786482, seqNum=-1] 2024-11-22T03:41:50,796 DEBUG [Thread-1809 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-22T03:41:50,796 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41012, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-22T03:41:50,797 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41022, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-22T03:41:50,797 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2] 2024-11-22T03:41:50,798 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2] 2024-11-22T03:41:50,800 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-22T03:41:50,801 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4a7ce0cfb17f4dcc3dd24b2156963795 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-22T03:41:50,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(8855): Flush requested on 4a7ce0cfb17f4dcc3dd24b2156963795 2024-11-22T03:41:50,825 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/.tmp/A/ac88f1cb9e334b71884220ec66c56be9 is 50, key is test_row_0/A:col10/1732246910799/Put/seqid=0 2024-11-22T03:41:50,837 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742220_1396 (size=12001) 2024-11-22T03:41:50,848 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:50,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40962 deadline: 1732246970840, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:50,849 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:50,849 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:50,849 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:50,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40990 deadline: 1732246970840, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:50,849 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:41:50,849 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:50,849 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:50,849 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:50,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40926 deadline: 1732246970846, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:50,849 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:41:50,849 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:50,849 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:50,849 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:41:50,854 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:50,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40966 deadline: 1732246970847, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:50,855 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:50,855 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:50,855 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:41:50,855 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:50,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41022 deadline: 1732246970848, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:50,856 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:50,856 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:50,856 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:41:50,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-22T03:41:50,947 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42545 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-22T03:41:50,948 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795. 2024-11-22T03:41:50,948 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795. as already flushing 2024-11-22T03:41:50,948 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795. 2024-11-22T03:41:50,948 ERROR [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 4a7ce0cfb17f4dcc3dd24b2156963795, NAME => 'TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:41:50,948 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 4a7ce0cfb17f4dcc3dd24b2156963795, NAME => 'TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:41:50,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42627 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4a7ce0cfb17f4dcc3dd24b2156963795, NAME => 'TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4a7ce0cfb17f4dcc3dd24b2156963795, NAME => 'TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:41:50,963 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:50,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40962 deadline: 1732246970958, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:50,963 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:50,963 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:50,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40990 deadline: 1732246970958, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:50,963 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:50,964 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:41:50,964 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:50,964 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:50,964 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:50,964 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:41:50,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40926 deadline: 1732246970958, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:50,964 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:50,964 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:50,964 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:50,964 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:41:50,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41022 deadline: 1732246970958, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:50,964 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:50,964 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:50,964 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:50,964 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:41:50,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40966 deadline: 1732246970958, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:50,965 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:50,965 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:50,965 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:41:51,100 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42545 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-22T03:41:51,100 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795. 2024-11-22T03:41:51,100 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795. as already flushing 2024-11-22T03:41:51,100 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795. 2024-11-22T03:41:51,100 ERROR [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 4a7ce0cfb17f4dcc3dd24b2156963795, NAME => 'TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:41:51,100 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 4a7ce0cfb17f4dcc3dd24b2156963795, NAME => 'TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:41:51,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42627 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4a7ce0cfb17f4dcc3dd24b2156963795, NAME => 'TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4a7ce0cfb17f4dcc3dd24b2156963795, NAME => 'TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:41:51,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-22T03:41:51,168 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:51,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40962 deadline: 1732246971168, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:51,169 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:51,169 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:51,169 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:51,169 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:41:51,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40990 deadline: 1732246971168, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:51,169 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:51,169 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:51,169 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:41:51,169 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:51,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40926 deadline: 1732246971168, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:51,170 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:51,170 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:51,170 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:41:51,170 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:51,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41022 deadline: 1732246971168, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:51,170 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:51,170 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:51,170 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:41:51,170 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:51,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40966 deadline: 1732246971169, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:51,170 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:51,170 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:51,170 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:41:51,238 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/.tmp/A/ac88f1cb9e334b71884220ec66c56be9 2024-11-22T03:41:51,253 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42545 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-22T03:41:51,253 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795. 2024-11-22T03:41:51,253 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795. as already flushing 2024-11-22T03:41:51,253 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795. 2024-11-22T03:41:51,253 ERROR [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 4a7ce0cfb17f4dcc3dd24b2156963795, NAME => 'TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:41:51,253 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 4a7ce0cfb17f4dcc3dd24b2156963795, NAME => 'TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:41:51,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42627 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4a7ce0cfb17f4dcc3dd24b2156963795, NAME => 'TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4a7ce0cfb17f4dcc3dd24b2156963795, NAME => 'TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:41:51,272 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/.tmp/B/4bb69e136b8c47268a9c8a395c02484b is 50, key is test_row_0/B:col10/1732246910799/Put/seqid=0 2024-11-22T03:41:51,286 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742221_1397 (size=12001) 2024-11-22T03:41:51,287 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/.tmp/B/4bb69e136b8c47268a9c8a395c02484b 2024-11-22T03:41:51,313 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/.tmp/C/6a0e4aa494e8439ba1cdc81dc334cf21 is 50, key is test_row_0/C:col10/1732246910799/Put/seqid=0 2024-11-22T03:41:51,322 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742222_1398 (size=12001) 2024-11-22T03:41:51,324 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/.tmp/C/6a0e4aa494e8439ba1cdc81dc334cf21 2024-11-22T03:41:51,330 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/.tmp/A/ac88f1cb9e334b71884220ec66c56be9 as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/A/ac88f1cb9e334b71884220ec66c56be9 2024-11-22T03:41:51,335 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/A/ac88f1cb9e334b71884220ec66c56be9, entries=150, sequenceid=13, filesize=11.7 K 2024-11-22T03:41:51,336 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/.tmp/B/4bb69e136b8c47268a9c8a395c02484b as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/B/4bb69e136b8c47268a9c8a395c02484b 2024-11-22T03:41:51,345 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/B/4bb69e136b8c47268a9c8a395c02484b, entries=150, sequenceid=13, filesize=11.7 K 2024-11-22T03:41:51,346 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/.tmp/C/6a0e4aa494e8439ba1cdc81dc334cf21 as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/C/6a0e4aa494e8439ba1cdc81dc334cf21 2024-11-22T03:41:51,353 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/C/6a0e4aa494e8439ba1cdc81dc334cf21, entries=150, sequenceid=13, filesize=11.7 K 2024-11-22T03:41:51,355 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 4a7ce0cfb17f4dcc3dd24b2156963795 in 555ms, sequenceid=13, compaction requested=false 2024-11-22T03:41:51,355 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4a7ce0cfb17f4dcc3dd24b2156963795: 2024-11-22T03:41:51,407 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42545 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-22T03:41:51,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795. 2024-11-22T03:41:51,408 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2902): Flushing 4a7ce0cfb17f4dcc3dd24b2156963795 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-22T03:41:51,411 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/.tmp/A/22690dbf6bed459ca430a14060adfafe is 50, key is test_row_0/A:col10/1732246910846/Put/seqid=0 2024-11-22T03:41:51,415 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742223_1399 (size=12001) 2024-11-22T03:41:51,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-22T03:41:51,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(8855): Flush requested on 4a7ce0cfb17f4dcc3dd24b2156963795 2024-11-22T03:41:51,484 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795. as already flushing 2024-11-22T03:41:51,495 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:51,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40966 deadline: 1732246971488, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:51,496 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:51,496 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:51,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40990 deadline: 1732246971492, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:51,496 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:51,496 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:41:51,496 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:51,496 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:51,496 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:41:51,500 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:51,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40962 deadline: 1732246971493, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:51,500 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:51,500 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:51,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40926 deadline: 1732246971495, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:51,500 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:51,500 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:41:51,500 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:51,500 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:51,500 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:41:51,502 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:51,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41022 deadline: 1732246971496, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:51,503 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:51,503 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:51,503 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:41:51,599 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:51,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40966 deadline: 1732246971599, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:51,600 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:51,600 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:51,600 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:51,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40990 deadline: 1732246971599, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:51,600 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:41:51,600 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:51,600 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:51,600 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:41:51,611 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:51,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40962 deadline: 1732246971608, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:51,611 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:51,611 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:51,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40926 deadline: 1732246971608, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:51,611 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:51,611 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:41:51,612 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:51,612 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:51,612 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:51,612 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:41:51,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41022 deadline: 1732246971608, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:51,612 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:51,612 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:51,612 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:41:51,813 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:51,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40990 deadline: 1732246971809, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:51,814 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:51,814 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:51,814 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:51,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40966 deadline: 1732246971809, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:51,814 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:41:51,814 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:51,814 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:51,814 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:41:51,816 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/.tmp/A/22690dbf6bed459ca430a14060adfafe 2024-11-22T03:41:51,821 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:51,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40926 deadline: 1732246971819, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:51,822 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:51,822 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:51,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41022 deadline: 1732246971819, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:51,822 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:51,822 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:41:51,822 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:51,822 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:51,822 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:51,822 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:41:51,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40962 deadline: 1732246971819, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:51,823 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/.tmp/B/a74e1c34d9f941db99f8b7cc570fd51c is 50, key is test_row_0/B:col10/1732246910846/Put/seqid=0 2024-11-22T03:41:51,823 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:51,823 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:51,823 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:41:51,825 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742224_1400 (size=12001) 2024-11-22T03:41:51,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-22T03:41:52,121 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:52,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40990 deadline: 1732246972119, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:52,121 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:52,121 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:52,122 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:41:52,127 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:52,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40966 deadline: 1732246972121, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:52,130 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:52,130 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:52,130 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:41:52,130 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:52,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41022 deadline: 1732246972129, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:52,131 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:52,131 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:52,131 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:52,131 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:41:52,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40926 deadline: 1732246972129, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:52,131 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:52,131 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:52,131 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:52,131 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:41:52,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40962 deadline: 1732246972129, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:52,131 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:52,131 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:52,131 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:41:52,226 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/.tmp/B/a74e1c34d9f941db99f8b7cc570fd51c 2024-11-22T03:41:52,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/.tmp/C/4e4851c784944e6482dec1e32d0e2ee3 is 50, key is test_row_0/C:col10/1732246910846/Put/seqid=0 2024-11-22T03:41:52,248 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742225_1401 (size=12001) 2024-11-22T03:41:52,248 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/.tmp/C/4e4851c784944e6482dec1e32d0e2ee3 2024-11-22T03:41:52,255 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/.tmp/A/22690dbf6bed459ca430a14060adfafe as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/A/22690dbf6bed459ca430a14060adfafe 2024-11-22T03:41:52,260 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/A/22690dbf6bed459ca430a14060adfafe, entries=150, sequenceid=37, filesize=11.7 K 2024-11-22T03:41:52,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/.tmp/B/a74e1c34d9f941db99f8b7cc570fd51c as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/B/a74e1c34d9f941db99f8b7cc570fd51c 2024-11-22T03:41:52,265 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/B/a74e1c34d9f941db99f8b7cc570fd51c, entries=150, sequenceid=37, filesize=11.7 K 2024-11-22T03:41:52,266 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/.tmp/C/4e4851c784944e6482dec1e32d0e2ee3 as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/C/4e4851c784944e6482dec1e32d0e2ee3 2024-11-22T03:41:52,273 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/C/4e4851c784944e6482dec1e32d0e2ee3, entries=150, sequenceid=37, filesize=11.7 K 2024-11-22T03:41:52,274 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for 4a7ce0cfb17f4dcc3dd24b2156963795 in 867ms, sequenceid=37, compaction requested=false 2024-11-22T03:41:52,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2603): Flush status journal for 4a7ce0cfb17f4dcc3dd24b2156963795: 2024-11-22T03:41:52,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795. 2024-11-22T03:41:52,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=118 2024-11-22T03:41:52,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42627 {}] master.HMaster(4169): Remote procedure done, pid=118 2024-11-22T03:41:52,277 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=118, resume processing ppid=117 2024-11-22T03:41:52,277 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=118, ppid=117, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4840 sec 2024-11-22T03:41:52,279 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=117, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees in 1.4950 sec 2024-11-22T03:41:52,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(8855): Flush requested on 4a7ce0cfb17f4dcc3dd24b2156963795 2024-11-22T03:41:52,631 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-22T03:41:52,632 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4a7ce0cfb17f4dcc3dd24b2156963795 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-22T03:41:52,635 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/.tmp/A/71277d86252c497a98179237e4977b01 is 50, key is test_row_0/A:col10/1732246912630/Put/seqid=0 2024-11-22T03:41:52,638 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742226_1402 (size=14341) 2024-11-22T03:41:52,639 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/.tmp/A/71277d86252c497a98179237e4977b01 2024-11-22T03:41:52,646 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/.tmp/B/f283a1d9d4e7406fb7aeaffbe3e43f82 is 50, key is test_row_0/B:col10/1732246912630/Put/seqid=0 2024-11-22T03:41:52,649 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742227_1403 (size=12001) 2024-11-22T03:41:52,662 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-22T03:41:52,678 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:52,678 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:52,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40990 deadline: 1732246972667, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:52,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41022 deadline: 1732246972666, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:52,678 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:52,678 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:52,678 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:52,678 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:52,678 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:41:52,678 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:41:52,678 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:52,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40966 deadline: 1732246972668, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:52,679 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:52,679 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:52,679 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:41:52,686 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:52,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40926 deadline: 1732246972678, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:52,687 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:52,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40962 deadline: 1732246972678, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:52,687 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:52,687 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:52,687 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:41:52,687 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:52,687 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:52,687 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:41:52,791 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:52,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40990 deadline: 1732246972789, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:52,791 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:52,792 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:52,791 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:52,792 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:41:52,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41022 deadline: 1732246972789, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:52,792 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:52,792 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:52,792 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:52,792 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:41:52,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40966 deadline: 1732246972789, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:52,792 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:52,792 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:52,792 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:52,792 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:41:52,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40926 deadline: 1732246972789, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:52,792 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:52,792 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:52,792 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:41:52,804 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:52,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40962 deadline: 1732246972798, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:52,805 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:52,805 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:52,805 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:41:52,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-22T03:41:52,939 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-22T03:41:52,939 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42627 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-22T03:41:52,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42627 {}] procedure2.ProcedureExecutor(1139): Stored pid=119, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees 2024-11-22T03:41:52,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-22T03:41:52,940 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=119, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-22T03:41:52,941 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=119, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-22T03:41:52,941 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=120, ppid=119, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-22T03:41:53,005 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:53,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40990 deadline: 1732246972999, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:53,006 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:53,006 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:53,006 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:53,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41022 deadline: 1732246972999, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:53,006 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:41:53,006 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:53,006 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:53,006 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:41:53,006 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:53,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40966 deadline: 1732246972999, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:53,006 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:53,006 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:53,006 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:53,006 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:41:53,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40926 deadline: 1732246972999, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:53,006 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:53,007 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:53,007 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:41:53,014 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:53,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40962 deadline: 1732246973008, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:53,015 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:53,015 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:53,015 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:41:53,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-22T03:41:53,050 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/.tmp/B/f283a1d9d4e7406fb7aeaffbe3e43f82 2024-11-22T03:41:53,055 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/.tmp/C/3d1667e5fc9749ccbf1b0b878518eb09 is 50, key is test_row_0/C:col10/1732246912630/Put/seqid=0 2024-11-22T03:41:53,060 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742228_1404 (size=12001) 2024-11-22T03:41:53,093 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42545 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-22T03:41:53,093 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795. 2024-11-22T03:41:53,093 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795. as already flushing 2024-11-22T03:41:53,093 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795. 2024-11-22T03:41:53,093 ERROR [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 4a7ce0cfb17f4dcc3dd24b2156963795, NAME => 'TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:41:53,093 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 4a7ce0cfb17f4dcc3dd24b2156963795, NAME => 'TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:41:53,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42627 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4a7ce0cfb17f4dcc3dd24b2156963795, NAME => 'TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4a7ce0cfb17f4dcc3dd24b2156963795, NAME => 'TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:41:53,245 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42545 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-22T03:41:53,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795. 2024-11-22T03:41:53,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795. as already flushing 2024-11-22T03:41:53,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795. 2024-11-22T03:41:53,246 ERROR [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 4a7ce0cfb17f4dcc3dd24b2156963795, NAME => 'TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:41:53,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 4a7ce0cfb17f4dcc3dd24b2156963795, NAME => 'TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:41:53,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42627 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4a7ce0cfb17f4dcc3dd24b2156963795, NAME => 'TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4a7ce0cfb17f4dcc3dd24b2156963795, NAME => 'TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:41:53,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-22T03:41:53,318 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:53,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40990 deadline: 1732246973308, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:53,318 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:53,318 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:53,318 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:53,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40966 deadline: 1732246973308, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:53,318 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:41:53,318 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:53,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40926 deadline: 1732246973308, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:53,319 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:53,319 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:53,319 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:41:53,319 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:53,319 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:53,319 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:41:53,325 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:53,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41022 deadline: 1732246973319, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:53,325 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:53,325 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:53,325 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:53,325 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:41:53,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40962 deadline: 1732246973319, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:53,325 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:53,325 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:53,325 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:41:53,399 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42545 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-22T03:41:53,399 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795. 2024-11-22T03:41:53,399 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795. as already flushing 2024-11-22T03:41:53,399 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795. 2024-11-22T03:41:53,399 ERROR [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 4a7ce0cfb17f4dcc3dd24b2156963795, NAME => 'TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:41:53,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 4a7ce0cfb17f4dcc3dd24b2156963795, NAME => 'TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:41:53,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42627 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4a7ce0cfb17f4dcc3dd24b2156963795, NAME => 'TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4a7ce0cfb17f4dcc3dd24b2156963795, NAME => 'TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:41:53,461 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/.tmp/C/3d1667e5fc9749ccbf1b0b878518eb09 2024-11-22T03:41:53,469 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/.tmp/A/71277d86252c497a98179237e4977b01 as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/A/71277d86252c497a98179237e4977b01 2024-11-22T03:41:53,473 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/A/71277d86252c497a98179237e4977b01, entries=200, sequenceid=50, filesize=14.0 K 2024-11-22T03:41:53,475 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/.tmp/B/f283a1d9d4e7406fb7aeaffbe3e43f82 as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/B/f283a1d9d4e7406fb7aeaffbe3e43f82 2024-11-22T03:41:53,479 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/B/f283a1d9d4e7406fb7aeaffbe3e43f82, entries=150, sequenceid=50, filesize=11.7 K 2024-11-22T03:41:53,480 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/.tmp/C/3d1667e5fc9749ccbf1b0b878518eb09 as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/C/3d1667e5fc9749ccbf1b0b878518eb09 2024-11-22T03:41:53,483 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/C/3d1667e5fc9749ccbf1b0b878518eb09, entries=150, sequenceid=50, filesize=11.7 K 2024-11-22T03:41:53,484 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for 4a7ce0cfb17f4dcc3dd24b2156963795 in 853ms, sequenceid=50, compaction requested=true 2024-11-22T03:41:53,484 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4a7ce0cfb17f4dcc3dd24b2156963795: 2024-11-22T03:41:53,484 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4a7ce0cfb17f4dcc3dd24b2156963795:A, priority=-2147483648, current under compaction store size is 1 2024-11-22T03:41:53,484 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T03:41:53,484 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T03:41:53,484 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T03:41:53,484 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4a7ce0cfb17f4dcc3dd24b2156963795:B, priority=-2147483648, current under compaction store size is 2 2024-11-22T03:41:53,484 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T03:41:53,485 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4a7ce0cfb17f4dcc3dd24b2156963795:C, priority=-2147483648, current under compaction store size is 3 2024-11-22T03:41:53,485 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T03:41:53,485 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38343 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T03:41:53,485 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T03:41:53,485 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.HStore(1541): 4a7ce0cfb17f4dcc3dd24b2156963795/A is initiating minor compaction (all files) 2024-11-22T03:41:53,485 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HStore(1541): 4a7ce0cfb17f4dcc3dd24b2156963795/B is initiating minor compaction (all files) 2024-11-22T03:41:53,485 INFO [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4a7ce0cfb17f4dcc3dd24b2156963795/A in TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795. 2024-11-22T03:41:53,485 INFO [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4a7ce0cfb17f4dcc3dd24b2156963795/B in TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795. 2024-11-22T03:41:53,485 INFO [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/A/ac88f1cb9e334b71884220ec66c56be9, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/A/22690dbf6bed459ca430a14060adfafe, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/A/71277d86252c497a98179237e4977b01] into tmpdir=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/.tmp, totalSize=37.4 K 2024-11-22T03:41:53,485 INFO [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/B/4bb69e136b8c47268a9c8a395c02484b, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/B/a74e1c34d9f941db99f8b7cc570fd51c, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/B/f283a1d9d4e7406fb7aeaffbe3e43f82] into tmpdir=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/.tmp, totalSize=35.2 K 2024-11-22T03:41:53,485 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] compactions.Compactor(225): Compacting 4bb69e136b8c47268a9c8a395c02484b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1732246910798 2024-11-22T03:41:53,485 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] compactions.Compactor(225): Compacting ac88f1cb9e334b71884220ec66c56be9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1732246910798 2024-11-22T03:41:53,486 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] compactions.Compactor(225): Compacting a74e1c34d9f941db99f8b7cc570fd51c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1732246910839 2024-11-22T03:41:53,486 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] compactions.Compactor(225): Compacting 22690dbf6bed459ca430a14060adfafe, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1732246910839 2024-11-22T03:41:53,488 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] compactions.Compactor(225): Compacting 71277d86252c497a98179237e4977b01, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1732246911484 2024-11-22T03:41:53,488 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] compactions.Compactor(225): Compacting f283a1d9d4e7406fb7aeaffbe3e43f82, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1732246911484 2024-11-22T03:41:53,495 INFO [RS:0;c85114ed5096:42545-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4a7ce0cfb17f4dcc3dd24b2156963795#A#compaction#339 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T03:41:53,496 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/.tmp/A/d54a9efc230f472e9418bc21f61f0f69 is 50, key is test_row_0/A:col10/1732246912630/Put/seqid=0 2024-11-22T03:41:53,503 INFO [RS:0;c85114ed5096:42545-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4a7ce0cfb17f4dcc3dd24b2156963795#B#compaction#340 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T03:41:53,503 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/.tmp/B/1d942119e672441586705152a7d45a23 is 50, key is test_row_0/B:col10/1732246912630/Put/seqid=0 2024-11-22T03:41:53,510 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742229_1405 (size=12104) 2024-11-22T03:41:53,511 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742230_1406 (size=12104) 2024-11-22T03:41:53,513 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/.tmp/A/d54a9efc230f472e9418bc21f61f0f69 as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/A/d54a9efc230f472e9418bc21f61f0f69 2024-11-22T03:41:53,518 INFO [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4a7ce0cfb17f4dcc3dd24b2156963795/A of 4a7ce0cfb17f4dcc3dd24b2156963795 into d54a9efc230f472e9418bc21f61f0f69(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T03:41:53,518 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4a7ce0cfb17f4dcc3dd24b2156963795: 2024-11-22T03:41:53,518 INFO [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., storeName=4a7ce0cfb17f4dcc3dd24b2156963795/A, priority=13, startTime=1732246913484; duration=0sec 2024-11-22T03:41:53,518 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T03:41:53,518 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4a7ce0cfb17f4dcc3dd24b2156963795:A 2024-11-22T03:41:53,518 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T03:41:53,519 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T03:41:53,519 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.HStore(1541): 4a7ce0cfb17f4dcc3dd24b2156963795/C is initiating minor compaction (all files) 2024-11-22T03:41:53,519 INFO [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4a7ce0cfb17f4dcc3dd24b2156963795/C in TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795. 2024-11-22T03:41:53,519 INFO [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/C/6a0e4aa494e8439ba1cdc81dc334cf21, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/C/4e4851c784944e6482dec1e32d0e2ee3, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/C/3d1667e5fc9749ccbf1b0b878518eb09] into tmpdir=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/.tmp, totalSize=35.2 K 2024-11-22T03:41:53,519 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6a0e4aa494e8439ba1cdc81dc334cf21, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1732246910798 2024-11-22T03:41:53,520 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4e4851c784944e6482dec1e32d0e2ee3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1732246910839 2024-11-22T03:41:53,520 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3d1667e5fc9749ccbf1b0b878518eb09, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1732246911484 2024-11-22T03:41:53,527 INFO [RS:0;c85114ed5096:42545-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4a7ce0cfb17f4dcc3dd24b2156963795#C#compaction#341 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T03:41:53,527 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/.tmp/C/59061dfed12c47f8a8d84488fa51ddc9 is 50, key is test_row_0/C:col10/1732246912630/Put/seqid=0 2024-11-22T03:41:53,534 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742231_1407 (size=12104) 2024-11-22T03:41:53,544 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/.tmp/C/59061dfed12c47f8a8d84488fa51ddc9 as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/C/59061dfed12c47f8a8d84488fa51ddc9 2024-11-22T03:41:53,550 INFO [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4a7ce0cfb17f4dcc3dd24b2156963795/C of 4a7ce0cfb17f4dcc3dd24b2156963795 into 59061dfed12c47f8a8d84488fa51ddc9(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T03:41:53,550 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4a7ce0cfb17f4dcc3dd24b2156963795: 2024-11-22T03:41:53,550 INFO [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., storeName=4a7ce0cfb17f4dcc3dd24b2156963795/C, priority=13, startTime=1732246913484; duration=0sec 2024-11-22T03:41:53,550 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T03:41:53,550 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4a7ce0cfb17f4dcc3dd24b2156963795:C 2024-11-22T03:41:53,552 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42545 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-22T03:41:53,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795. 2024-11-22T03:41:53,552 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2902): Flushing 4a7ce0cfb17f4dcc3dd24b2156963795 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-22T03:41:53,556 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/.tmp/A/b8f71668dd8c40b39cd0500452b61a80 is 50, key is test_row_0/A:col10/1732246912677/Put/seqid=0 2024-11-22T03:41:53,566 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742232_1408 (size=12001) 2024-11-22T03:41:53,567 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=75 (bloomFilter=true), to=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/.tmp/A/b8f71668dd8c40b39cd0500452b61a80 2024-11-22T03:41:53,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-22T03:41:53,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/.tmp/B/a55c23f912144f2c97cec3b2f8acf963 is 50, key is test_row_0/B:col10/1732246912677/Put/seqid=0 2024-11-22T03:41:53,596 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742233_1409 (size=12001) 2024-11-22T03:41:53,835 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795. as already flushing 2024-11-22T03:41:53,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(8855): Flush requested on 4a7ce0cfb17f4dcc3dd24b2156963795 2024-11-22T03:41:53,851 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:53,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41022 deadline: 1732246973848, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:53,852 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:53,852 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:53,852 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:41:53,854 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:53,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40990 deadline: 1732246973849, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:53,854 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:53,854 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:53,854 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:53,854 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:41:53,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40962 deadline: 1732246973850, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:53,855 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:53,855 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:53,855 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:53,855 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:41:53,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40926 deadline: 1732246973851, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:53,855 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:53,855 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:53,855 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:41:53,862 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:53,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40966 deadline: 1732246973853, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:53,863 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:53,863 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:53,863 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:41:53,916 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/.tmp/B/1d942119e672441586705152a7d45a23 as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/B/1d942119e672441586705152a7d45a23 2024-11-22T03:41:53,920 INFO [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4a7ce0cfb17f4dcc3dd24b2156963795/B of 4a7ce0cfb17f4dcc3dd24b2156963795 into 1d942119e672441586705152a7d45a23(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T03:41:53,920 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4a7ce0cfb17f4dcc3dd24b2156963795: 2024-11-22T03:41:53,920 INFO [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., storeName=4a7ce0cfb17f4dcc3dd24b2156963795/B, priority=13, startTime=1732246913484; duration=0sec 2024-11-22T03:41:53,920 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T03:41:53,920 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4a7ce0cfb17f4dcc3dd24b2156963795:B 2024-11-22T03:41:53,960 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:53,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41022 deadline: 1732246973958, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:53,960 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:53,960 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:53,960 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:53,960 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:41:53,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40990 deadline: 1732246973959, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:53,961 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:53,961 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:53,961 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:53,961 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:41:53,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40962 deadline: 1732246973959, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:53,961 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:53,961 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:53,961 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:41:53,961 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:53,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40926 deadline: 1732246973959, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:53,961 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:53,961 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:53,961 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:41:53,973 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:53,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40966 deadline: 1732246973968, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:53,973 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:53,973 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:53,973 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:41:53,997 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=75 (bloomFilter=true), to=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/.tmp/B/a55c23f912144f2c97cec3b2f8acf963 2024-11-22T03:41:54,003 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/.tmp/C/92a056a0d4ed4bceb6f96e9dc2bc8132 is 50, key is test_row_0/C:col10/1732246912677/Put/seqid=0 2024-11-22T03:41:54,014 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742234_1410 (size=12001) 2024-11-22T03:41:54,015 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=75 (bloomFilter=true), to=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/.tmp/C/92a056a0d4ed4bceb6f96e9dc2bc8132 2024-11-22T03:41:54,022 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/.tmp/A/b8f71668dd8c40b39cd0500452b61a80 as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/A/b8f71668dd8c40b39cd0500452b61a80 2024-11-22T03:41:54,025 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/A/b8f71668dd8c40b39cd0500452b61a80, entries=150, sequenceid=75, filesize=11.7 K 2024-11-22T03:41:54,026 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/.tmp/B/a55c23f912144f2c97cec3b2f8acf963 as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/B/a55c23f912144f2c97cec3b2f8acf963 2024-11-22T03:41:54,029 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/B/a55c23f912144f2c97cec3b2f8acf963, entries=150, sequenceid=75, filesize=11.7 K 2024-11-22T03:41:54,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/.tmp/C/92a056a0d4ed4bceb6f96e9dc2bc8132 as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/C/92a056a0d4ed4bceb6f96e9dc2bc8132 2024-11-22T03:41:54,032 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/C/92a056a0d4ed4bceb6f96e9dc2bc8132, entries=150, sequenceid=75, filesize=11.7 K 2024-11-22T03:41:54,033 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for 4a7ce0cfb17f4dcc3dd24b2156963795 in 481ms, sequenceid=75, compaction requested=false 2024-11-22T03:41:54,033 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2603): Flush status journal for 4a7ce0cfb17f4dcc3dd24b2156963795: 2024-11-22T03:41:54,033 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795. 2024-11-22T03:41:54,033 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=120 2024-11-22T03:41:54,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42627 {}] master.HMaster(4169): Remote procedure done, pid=120 2024-11-22T03:41:54,036 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=120, resume processing ppid=119 2024-11-22T03:41:54,036 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=120, ppid=119, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0930 sec 2024-11-22T03:41:54,040 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=119, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees in 1.1000 sec 2024-11-22T03:41:54,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-22T03:41:54,078 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-22T03:41:54,079 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42627 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-22T03:41:54,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42627 {}] procedure2.ProcedureExecutor(1139): Stored pid=121, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees 2024-11-22T03:41:54,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-22T03:41:54,081 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=121, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-22T03:41:54,081 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=121, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-22T03:41:54,081 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=122, ppid=121, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-22T03:41:54,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(8855): Flush requested on 4a7ce0cfb17f4dcc3dd24b2156963795 2024-11-22T03:41:54,175 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-22T03:41:54,175 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4a7ce0cfb17f4dcc3dd24b2156963795 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-22T03:41:54,179 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/.tmp/A/98f76382f1b64743ba793f18db616218 is 50, key is test_row_0/A:col10/1732246914174/Put/seqid=0 2024-11-22T03:41:54,186 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742235_1411 (size=16681) 2024-11-22T03:41:54,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-22T03:41:54,212 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:54,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40990 deadline: 1732246974202, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:54,212 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:54,212 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:54,212 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:54,212 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:41:54,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40926 deadline: 1732246974203, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:54,213 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:54,213 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:54,213 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:41:54,219 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:54,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40962 deadline: 1732246974211, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:54,219 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:54,219 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:54,219 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:41:54,222 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:54,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40966 deadline: 1732246974212, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:54,222 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:54,222 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:54,222 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:41:54,223 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:54,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41022 deadline: 1732246974214, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:54,224 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:54,224 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:54,224 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:41:54,233 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42545 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-22T03:41:54,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795. 2024-11-22T03:41:54,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795. as already flushing 2024-11-22T03:41:54,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795. 2024-11-22T03:41:54,234 ERROR [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 4a7ce0cfb17f4dcc3dd24b2156963795, NAME => 'TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:41:54,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 4a7ce0cfb17f4dcc3dd24b2156963795, NAME => 'TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:41:54,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42627 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4a7ce0cfb17f4dcc3dd24b2156963795, NAME => 'TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4a7ce0cfb17f4dcc3dd24b2156963795, NAME => 'TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:41:54,323 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:54,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40990 deadline: 1732246974318, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:54,324 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:54,324 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:54,323 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:54,324 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:41:54,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40926 deadline: 1732246974319, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:54,324 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:54,324 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:54,324 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:41:54,334 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:54,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40962 deadline: 1732246974329, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:54,334 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:54,334 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:54,334 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:41:54,336 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:54,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40966 deadline: 1732246974329, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:54,336 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:54,336 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:54,336 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:54,336 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:41:54,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41022 deadline: 1732246974329, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:54,336 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:54,337 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:54,337 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:41:54,387 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42545 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-22T03:41:54,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795. 2024-11-22T03:41:54,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795. as already flushing 2024-11-22T03:41:54,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795. 2024-11-22T03:41:54,387 ERROR [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 4a7ce0cfb17f4dcc3dd24b2156963795, NAME => 'TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:41:54,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 4a7ce0cfb17f4dcc3dd24b2156963795, NAME => 'TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:41:54,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42627 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4a7ce0cfb17f4dcc3dd24b2156963795, NAME => 'TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4a7ce0cfb17f4dcc3dd24b2156963795, NAME => 'TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:41:54,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-22T03:41:54,531 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:54,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40990 deadline: 1732246974529, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:54,531 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:54,531 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:54,531 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:54,531 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:41:54,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40926 deadline: 1732246974529, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:54,532 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:54,532 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:54,532 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:41:54,540 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42545 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-22T03:41:54,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795. 2024-11-22T03:41:54,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795. as already flushing 2024-11-22T03:41:54,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795. 2024-11-22T03:41:54,540 ERROR [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 4a7ce0cfb17f4dcc3dd24b2156963795, NAME => 'TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:41:54,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 4a7ce0cfb17f4dcc3dd24b2156963795, NAME => 'TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:41:54,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42627 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4a7ce0cfb17f4dcc3dd24b2156963795, NAME => 'TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4a7ce0cfb17f4dcc3dd24b2156963795, NAME => 'TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:41:54,546 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:54,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40962 deadline: 1732246974538, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:54,546 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:54,546 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:54,546 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:41:54,554 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:54,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40966 deadline: 1732246974548, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:54,554 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:54,554 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:54,554 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:54,554 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:41:54,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41022 deadline: 1732246974548, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:54,555 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:54,555 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:54,555 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:41:54,586 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=91 (bloomFilter=true), to=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/.tmp/A/98f76382f1b64743ba793f18db616218 2024-11-22T03:41:54,592 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/.tmp/B/829066b589504d938b2fb6ac37a879b1 is 50, key is test_row_0/B:col10/1732246914174/Put/seqid=0 2024-11-22T03:41:54,595 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742236_1412 (size=12001) 2024-11-22T03:41:54,692 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42545 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-22T03:41:54,693 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795. 2024-11-22T03:41:54,693 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795. as already flushing 2024-11-22T03:41:54,693 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795. 2024-11-22T03:41:54,693 ERROR [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 4a7ce0cfb17f4dcc3dd24b2156963795, NAME => 'TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:41:54,693 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 4a7ce0cfb17f4dcc3dd24b2156963795, NAME => 'TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:41:54,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42627 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4a7ce0cfb17f4dcc3dd24b2156963795, NAME => 'TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4a7ce0cfb17f4dcc3dd24b2156963795, NAME => 'TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:41:54,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-22T03:41:54,844 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42545 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-22T03:41:54,844 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795. 2024-11-22T03:41:54,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795. as already flushing 2024-11-22T03:41:54,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795. 2024-11-22T03:41:54,845 ERROR [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 4a7ce0cfb17f4dcc3dd24b2156963795, NAME => 'TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:41:54,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 4a7ce0cfb17f4dcc3dd24b2156963795, NAME => 'TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:41:54,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42627 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4a7ce0cfb17f4dcc3dd24b2156963795, NAME => 'TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4a7ce0cfb17f4dcc3dd24b2156963795, NAME => 'TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:41:54,846 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:54,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40990 deadline: 1732246974838, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:54,846 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:54,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40926 deadline: 1732246974838, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:54,846 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:54,846 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:54,846 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:41:54,846 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:54,846 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:54,846 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:41:54,865 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:54,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40962 deadline: 1732246974859, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:54,866 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:54,866 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:54,866 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:54,866 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:41:54,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40966 deadline: 1732246974859, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:54,866 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:54,866 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:54,866 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:41:54,866 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:54,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41022 deadline: 1732246974859, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:54,866 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:54,866 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:54,866 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:41:54,996 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=91 (bloomFilter=true), to=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/.tmp/B/829066b589504d938b2fb6ac37a879b1 2024-11-22T03:41:54,996 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42545 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-22T03:41:54,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795. 2024-11-22T03:41:54,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795. as already flushing 2024-11-22T03:41:54,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795. 2024-11-22T03:41:54,997 ERROR [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 4a7ce0cfb17f4dcc3dd24b2156963795, NAME => 'TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:41:54,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 4a7ce0cfb17f4dcc3dd24b2156963795, NAME => 'TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:41:54,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42627 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4a7ce0cfb17f4dcc3dd24b2156963795, NAME => 'TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4a7ce0cfb17f4dcc3dd24b2156963795, NAME => 'TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:41:55,002 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/.tmp/C/864ce8fefe644bee8aa2e7d71cec23d2 is 50, key is test_row_0/C:col10/1732246914174/Put/seqid=0 2024-11-22T03:41:55,005 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742237_1413 (size=12001) 2024-11-22T03:41:55,150 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42545 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-22T03:41:55,150 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795. 2024-11-22T03:41:55,150 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795. as already flushing 2024-11-22T03:41:55,150 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795. 2024-11-22T03:41:55,150 ERROR [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 4a7ce0cfb17f4dcc3dd24b2156963795, NAME => 'TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:41:55,150 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 4a7ce0cfb17f4dcc3dd24b2156963795, NAME => 'TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:41:55,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42627 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4a7ce0cfb17f4dcc3dd24b2156963795, NAME => 'TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4a7ce0cfb17f4dcc3dd24b2156963795, NAME => 'TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:41:55,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-22T03:41:55,303 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42545 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-22T03:41:55,303 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795. 2024-11-22T03:41:55,303 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795. as already flushing 2024-11-22T03:41:55,303 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795. 2024-11-22T03:41:55,303 ERROR [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 4a7ce0cfb17f4dcc3dd24b2156963795, NAME => 'TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:41:55,303 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 4a7ce0cfb17f4dcc3dd24b2156963795, NAME => 'TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:41:55,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42627 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4a7ce0cfb17f4dcc3dd24b2156963795, NAME => 'TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4a7ce0cfb17f4dcc3dd24b2156963795, NAME => 'TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:41:55,364 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:55,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40990 deadline: 1732246975359, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:55,364 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:55,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40926 deadline: 1732246975359, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:55,365 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:55,365 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:55,365 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:41:55,365 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:55,365 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:55,365 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:41:55,374 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:55,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40962 deadline: 1732246975369, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:55,375 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:55,375 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:55,375 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:41:55,384 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:55,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40966 deadline: 1732246975378, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:55,385 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:55,385 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:55,385 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:55,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41022 deadline: 1732246975378, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:55,385 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:41:55,385 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:55,385 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:55,385 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:41:55,406 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=91 (bloomFilter=true), to=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/.tmp/C/864ce8fefe644bee8aa2e7d71cec23d2 2024-11-22T03:41:55,412 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/.tmp/A/98f76382f1b64743ba793f18db616218 as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/A/98f76382f1b64743ba793f18db616218 2024-11-22T03:41:55,415 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/A/98f76382f1b64743ba793f18db616218, entries=250, sequenceid=91, filesize=16.3 K 2024-11-22T03:41:55,416 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/.tmp/B/829066b589504d938b2fb6ac37a879b1 as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/B/829066b589504d938b2fb6ac37a879b1 2024-11-22T03:41:55,419 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/B/829066b589504d938b2fb6ac37a879b1, entries=150, sequenceid=91, filesize=11.7 K 2024-11-22T03:41:55,419 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/.tmp/C/864ce8fefe644bee8aa2e7d71cec23d2 as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/C/864ce8fefe644bee8aa2e7d71cec23d2 2024-11-22T03:41:55,422 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/C/864ce8fefe644bee8aa2e7d71cec23d2, entries=150, sequenceid=91, filesize=11.7 K 2024-11-22T03:41:55,423 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=120.76 KB/123660 for 4a7ce0cfb17f4dcc3dd24b2156963795 in 1248ms, sequenceid=91, compaction requested=true 2024-11-22T03:41:55,423 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4a7ce0cfb17f4dcc3dd24b2156963795: 2024-11-22T03:41:55,423 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T03:41:55,424 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4a7ce0cfb17f4dcc3dd24b2156963795:A, priority=-2147483648, current under compaction store size is 1 2024-11-22T03:41:55,424 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T03:41:55,424 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T03:41:55,424 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40786 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T03:41:55,424 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4a7ce0cfb17f4dcc3dd24b2156963795:B, priority=-2147483648, current under compaction store size is 2 2024-11-22T03:41:55,424 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T03:41:55,424 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.HStore(1541): 4a7ce0cfb17f4dcc3dd24b2156963795/A is initiating minor compaction (all files) 2024-11-22T03:41:55,424 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4a7ce0cfb17f4dcc3dd24b2156963795:C, priority=-2147483648, current under compaction store size is 3 2024-11-22T03:41:55,424 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T03:41:55,424 INFO [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4a7ce0cfb17f4dcc3dd24b2156963795/A in TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795. 2024-11-22T03:41:55,424 INFO [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/A/d54a9efc230f472e9418bc21f61f0f69, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/A/b8f71668dd8c40b39cd0500452b61a80, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/A/98f76382f1b64743ba793f18db616218] into tmpdir=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/.tmp, totalSize=39.8 K 2024-11-22T03:41:55,425 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] compactions.Compactor(225): Compacting d54a9efc230f472e9418bc21f61f0f69, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1732246911484 2024-11-22T03:41:55,425 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] compactions.Compactor(225): Compacting b8f71668dd8c40b39cd0500452b61a80, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=75, earliestPutTs=1732246912665 2024-11-22T03:41:55,425 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T03:41:55,425 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HStore(1541): 4a7ce0cfb17f4dcc3dd24b2156963795/B is initiating minor compaction (all files) 2024-11-22T03:41:55,425 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] compactions.Compactor(225): Compacting 98f76382f1b64743ba793f18db616218, keycount=250, bloomtype=ROW, size=16.3 K, encoding=NONE, compression=NONE, seqNum=91, earliestPutTs=1732246913848 2024-11-22T03:41:55,425 INFO [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4a7ce0cfb17f4dcc3dd24b2156963795/B in TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795. 2024-11-22T03:41:55,425 INFO [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/B/1d942119e672441586705152a7d45a23, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/B/a55c23f912144f2c97cec3b2f8acf963, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/B/829066b589504d938b2fb6ac37a879b1] into tmpdir=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/.tmp, totalSize=35.3 K 2024-11-22T03:41:55,426 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] compactions.Compactor(225): Compacting 1d942119e672441586705152a7d45a23, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1732246911484 2024-11-22T03:41:55,426 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] compactions.Compactor(225): Compacting a55c23f912144f2c97cec3b2f8acf963, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=75, earliestPutTs=1732246912665 2024-11-22T03:41:55,426 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] compactions.Compactor(225): Compacting 829066b589504d938b2fb6ac37a879b1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=91, earliestPutTs=1732246913850 2024-11-22T03:41:55,431 INFO [RS:0;c85114ed5096:42545-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4a7ce0cfb17f4dcc3dd24b2156963795#A#compaction#348 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T03:41:55,431 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/.tmp/A/94b11a7be9aa448885581b203cbb3c99 is 50, key is test_row_0/A:col10/1732246914174/Put/seqid=0 2024-11-22T03:41:55,434 INFO [RS:0;c85114ed5096:42545-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4a7ce0cfb17f4dcc3dd24b2156963795#B#compaction#349 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T03:41:55,435 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/.tmp/B/ac98440d556545e0ae9f8aa814757c9d is 50, key is test_row_0/B:col10/1732246914174/Put/seqid=0 2024-11-22T03:41:55,438 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742238_1414 (size=12207) 2024-11-22T03:41:55,443 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/.tmp/A/94b11a7be9aa448885581b203cbb3c99 as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/A/94b11a7be9aa448885581b203cbb3c99 2024-11-22T03:41:55,450 INFO [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4a7ce0cfb17f4dcc3dd24b2156963795/A of 4a7ce0cfb17f4dcc3dd24b2156963795 into 94b11a7be9aa448885581b203cbb3c99(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T03:41:55,450 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4a7ce0cfb17f4dcc3dd24b2156963795: 2024-11-22T03:41:55,450 INFO [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., storeName=4a7ce0cfb17f4dcc3dd24b2156963795/A, priority=13, startTime=1732246915423; duration=0sec 2024-11-22T03:41:55,450 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T03:41:55,450 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4a7ce0cfb17f4dcc3dd24b2156963795:A 2024-11-22T03:41:55,450 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T03:41:55,451 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T03:41:55,451 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.HStore(1541): 4a7ce0cfb17f4dcc3dd24b2156963795/C is initiating minor compaction (all files) 2024-11-22T03:41:55,451 INFO [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4a7ce0cfb17f4dcc3dd24b2156963795/C in TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795. 2024-11-22T03:41:55,451 INFO [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/C/59061dfed12c47f8a8d84488fa51ddc9, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/C/92a056a0d4ed4bceb6f96e9dc2bc8132, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/C/864ce8fefe644bee8aa2e7d71cec23d2] into tmpdir=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/.tmp, totalSize=35.3 K 2024-11-22T03:41:55,451 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] compactions.Compactor(225): Compacting 59061dfed12c47f8a8d84488fa51ddc9, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1732246911484 2024-11-22T03:41:55,451 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] compactions.Compactor(225): Compacting 92a056a0d4ed4bceb6f96e9dc2bc8132, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=75, earliestPutTs=1732246912665 2024-11-22T03:41:55,452 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] compactions.Compactor(225): Compacting 864ce8fefe644bee8aa2e7d71cec23d2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=91, earliestPutTs=1732246913850 2024-11-22T03:41:55,457 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42545 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-22T03:41:55,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795. 2024-11-22T03:41:55,457 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2902): Flushing 4a7ce0cfb17f4dcc3dd24b2156963795 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2024-11-22T03:41:55,466 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742239_1415 (size=12207) 2024-11-22T03:41:55,470 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/.tmp/B/ac98440d556545e0ae9f8aa814757c9d as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/B/ac98440d556545e0ae9f8aa814757c9d 2024-11-22T03:41:55,471 INFO [RS:0;c85114ed5096:42545-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4a7ce0cfb17f4dcc3dd24b2156963795#C#compaction#350 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T03:41:55,471 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/.tmp/C/e897aad97196496696b4d82b2f2b5e80 is 50, key is test_row_0/C:col10/1732246914174/Put/seqid=0 2024-11-22T03:41:55,477 INFO [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4a7ce0cfb17f4dcc3dd24b2156963795/B of 4a7ce0cfb17f4dcc3dd24b2156963795 into ac98440d556545e0ae9f8aa814757c9d(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T03:41:55,477 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4a7ce0cfb17f4dcc3dd24b2156963795: 2024-11-22T03:41:55,477 INFO [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., storeName=4a7ce0cfb17f4dcc3dd24b2156963795/B, priority=13, startTime=1732246915424; duration=0sec 2024-11-22T03:41:55,477 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T03:41:55,477 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4a7ce0cfb17f4dcc3dd24b2156963795:B 2024-11-22T03:41:55,480 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/.tmp/A/8d1d6eaa02cc4493ac2e1c802f395cb4 is 50, key is test_row_0/A:col10/1732246914211/Put/seqid=0 2024-11-22T03:41:55,494 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742241_1417 (size=12001) 2024-11-22T03:41:55,495 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=113 (bloomFilter=true), to=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/.tmp/A/8d1d6eaa02cc4493ac2e1c802f395cb4 2024-11-22T03:41:55,496 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742240_1416 (size=12207) 2024-11-22T03:41:55,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/.tmp/B/9390ed4698b340348b6f6a8156f9b8da is 50, key is test_row_0/B:col10/1732246914211/Put/seqid=0 2024-11-22T03:41:55,509 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742242_1418 (size=12001) 2024-11-22T03:41:55,899 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/.tmp/C/e897aad97196496696b4d82b2f2b5e80 as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/C/e897aad97196496696b4d82b2f2b5e80 2024-11-22T03:41:55,904 INFO [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4a7ce0cfb17f4dcc3dd24b2156963795/C of 4a7ce0cfb17f4dcc3dd24b2156963795 into e897aad97196496696b4d82b2f2b5e80(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T03:41:55,904 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4a7ce0cfb17f4dcc3dd24b2156963795: 2024-11-22T03:41:55,904 INFO [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., storeName=4a7ce0cfb17f4dcc3dd24b2156963795/C, priority=13, startTime=1732246915424; duration=0sec 2024-11-22T03:41:55,904 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T03:41:55,904 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4a7ce0cfb17f4dcc3dd24b2156963795:C 2024-11-22T03:41:55,910 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=113 (bloomFilter=true), to=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/.tmp/B/9390ed4698b340348b6f6a8156f9b8da 2024-11-22T03:41:55,932 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/.tmp/C/e79f25f1f6a04dde8e1c6fe870de6db4 is 50, key is test_row_0/C:col10/1732246914211/Put/seqid=0 2024-11-22T03:41:55,938 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742243_1419 (size=12001) 2024-11-22T03:41:55,938 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=113 (bloomFilter=true), to=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/.tmp/C/e79f25f1f6a04dde8e1c6fe870de6db4 2024-11-22T03:41:55,942 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/.tmp/A/8d1d6eaa02cc4493ac2e1c802f395cb4 as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/A/8d1d6eaa02cc4493ac2e1c802f395cb4 2024-11-22T03:41:55,947 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/A/8d1d6eaa02cc4493ac2e1c802f395cb4, entries=150, sequenceid=113, filesize=11.7 K 2024-11-22T03:41:55,948 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/.tmp/B/9390ed4698b340348b6f6a8156f9b8da as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/B/9390ed4698b340348b6f6a8156f9b8da 2024-11-22T03:41:55,956 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/B/9390ed4698b340348b6f6a8156f9b8da, entries=150, sequenceid=113, filesize=11.7 K 2024-11-22T03:41:55,957 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/.tmp/C/e79f25f1f6a04dde8e1c6fe870de6db4 as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/C/e79f25f1f6a04dde8e1c6fe870de6db4 2024-11-22T03:41:55,964 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/C/e79f25f1f6a04dde8e1c6fe870de6db4, entries=150, sequenceid=113, filesize=11.7 K 2024-11-22T03:41:55,965 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~317.11 KB/324720, currentSize=0 B/0 for 4a7ce0cfb17f4dcc3dd24b2156963795 in 508ms, sequenceid=113, compaction requested=false 2024-11-22T03:41:55,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2603): Flush status journal for 4a7ce0cfb17f4dcc3dd24b2156963795: 2024-11-22T03:41:55,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795. 2024-11-22T03:41:55,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=122 2024-11-22T03:41:55,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42627 {}] master.HMaster(4169): Remote procedure done, pid=122 2024-11-22T03:41:55,968 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=122, resume processing ppid=121 2024-11-22T03:41:55,969 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=122, ppid=121, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8860 sec 2024-11-22T03:41:55,970 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=121, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees in 1.8900 sec 2024-11-22T03:41:56,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-22T03:41:56,229 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-22T03:41:56,229 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42627 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-22T03:41:56,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42627 {}] procedure2.ProcedureExecutor(1139): Stored pid=123, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees 2024-11-22T03:41:56,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-22T03:41:56,231 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=123, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-22T03:41:56,231 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=123, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-22T03:41:56,231 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=124, ppid=123, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-22T03:41:56,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-22T03:41:56,383 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42545 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-22T03:41:56,383 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795. 2024-11-22T03:41:56,383 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2902): Flushing 4a7ce0cfb17f4dcc3dd24b2156963795 3/3 column families, dataSize=26.84 KB heapSize=71.06 KB 2024-11-22T03:41:56,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/.tmp/A/7310966b824440a9bb996c72dc5b5e1a is 50, key is test_row_0/A:col10/1732246916383/Put/seqid=0 2024-11-22T03:41:56,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(8855): Flush requested on 4a7ce0cfb17f4dcc3dd24b2156963795 2024-11-22T03:41:56,388 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795. as already flushing 2024-11-22T03:41:56,394 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742244_1420 (size=16681) 2024-11-22T03:41:56,430 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:56,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40962 deadline: 1732246976421, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:56,431 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:56,431 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:56,431 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:56,431 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:41:56,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40926 deadline: 1732246976422, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:56,431 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:56,431 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:56,431 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:41:56,434 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:56,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40990 deadline: 1732246976423, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:56,434 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:56,434 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:56,434 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:41:56,434 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:56,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40966 deadline: 1732246976429, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:56,435 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:56,435 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:56,435 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:41:56,439 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:56,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41022 deadline: 1732246976430, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:56,439 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:56,439 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:56,439 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:41:56,544 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:56,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40962 deadline: 1732246976539, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:56,544 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:56,544 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:56,544 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:56,544 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:41:56,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40926 deadline: 1732246976539, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:56,545 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:56,545 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:56,545 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:41:56,545 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:56,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40990 deadline: 1732246976539, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:56,545 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:56,545 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:56,545 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:56,545 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:41:56,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40966 deadline: 1732246976539, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:56,545 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:56,545 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:56,545 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:41:56,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-22T03:41:56,557 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:56,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41022 deadline: 1732246976548, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:56,557 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:56,557 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:56,557 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:41:56,756 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:56,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40962 deadline: 1732246976748, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:56,756 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:56,756 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:56,756 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:56,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40926 deadline: 1732246976748, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:56,756 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:41:56,756 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:56,756 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:56,756 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:56,756 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:41:56,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40990 deadline: 1732246976748, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:56,757 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:56,757 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:56,757 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:56,757 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:41:56,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40966 deadline: 1732246976748, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:56,757 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:56,757 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:56,757 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:41:56,775 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:56,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41022 deadline: 1732246976769, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:56,775 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:56,776 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:56,776 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:41:56,794 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=124 (bloomFilter=true), to=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/.tmp/A/7310966b824440a9bb996c72dc5b5e1a 2024-11-22T03:41:56,802 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/.tmp/B/26dc1f560f78442da4005f4162884cfe is 50, key is test_row_0/B:col10/1732246916383/Put/seqid=0 2024-11-22T03:41:56,811 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742245_1421 (size=12001) 2024-11-22T03:41:56,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-22T03:41:57,063 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:57,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40962 deadline: 1732246977058, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:57,063 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:57,063 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:57,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40926 deadline: 1732246977058, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:57,063 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:57,063 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:41:57,064 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:57,064 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:57,064 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:41:57,072 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:57,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40990 deadline: 1732246977068, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:57,073 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:57,073 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:57,073 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:57,073 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:41:57,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40966 deadline: 1732246977068, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:57,073 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:57,073 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:57,073 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:41:57,094 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:57,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41022 deadline: 1732246977089, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:57,095 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:57,095 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:57,095 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:41:57,212 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=124 (bloomFilter=true), to=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/.tmp/B/26dc1f560f78442da4005f4162884cfe 2024-11-22T03:41:57,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/.tmp/C/40a6b580d41747a581790e5e89fa6652 is 50, key is test_row_0/C:col10/1732246916383/Put/seqid=0 2024-11-22T03:41:57,231 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742246_1422 (size=12001) 2024-11-22T03:41:57,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-22T03:41:57,572 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:57,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40926 deadline: 1732246977569, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:57,572 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:57,572 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:57,572 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:41:57,580 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:57,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40962 deadline: 1732246977579, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:57,581 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:57,581 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:57,581 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:57,581 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:41:57,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40990 deadline: 1732246977579, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:57,581 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:57,581 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:57,581 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:57,581 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:41:57,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40966 deadline: 1732246977579, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:57,581 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:57,581 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:57,582 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:41:57,607 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:57,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41022 deadline: 1732246977598, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:57,608 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:57,608 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:57,608 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:41:57,632 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=124 (bloomFilter=true), to=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/.tmp/C/40a6b580d41747a581790e5e89fa6652 2024-11-22T03:41:57,635 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/.tmp/A/7310966b824440a9bb996c72dc5b5e1a as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/A/7310966b824440a9bb996c72dc5b5e1a 2024-11-22T03:41:57,638 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/A/7310966b824440a9bb996c72dc5b5e1a, entries=250, sequenceid=124, filesize=16.3 K 2024-11-22T03:41:57,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/.tmp/B/26dc1f560f78442da4005f4162884cfe as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/B/26dc1f560f78442da4005f4162884cfe 2024-11-22T03:41:57,641 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/B/26dc1f560f78442da4005f4162884cfe, entries=150, sequenceid=124, filesize=11.7 K 2024-11-22T03:41:57,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/.tmp/C/40a6b580d41747a581790e5e89fa6652 as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/C/40a6b580d41747a581790e5e89fa6652 2024-11-22T03:41:57,646 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/C/40a6b580d41747a581790e5e89fa6652, entries=150, sequenceid=124, filesize=11.7 K 2024-11-22T03:41:57,646 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~106.17 KB/108720, currentSize=161.02 KB/164880 for 4a7ce0cfb17f4dcc3dd24b2156963795 in 1263ms, sequenceid=124, compaction requested=true 2024-11-22T03:41:57,646 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2603): Flush status journal for 4a7ce0cfb17f4dcc3dd24b2156963795: 2024-11-22T03:41:57,646 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795. 2024-11-22T03:41:57,646 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=124 2024-11-22T03:41:57,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42627 {}] master.HMaster(4169): Remote procedure done, pid=124 2024-11-22T03:41:57,649 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=124, resume processing ppid=123 2024-11-22T03:41:57,650 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=124, ppid=123, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4170 sec 2024-11-22T03:41:57,651 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=123, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees in 1.4210 sec 2024-11-22T03:41:58,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-22T03:41:58,379 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-22T03:41:58,380 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42627 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-22T03:41:58,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42627 {}] procedure2.ProcedureExecutor(1139): Stored pid=125, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees 2024-11-22T03:41:58,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-22T03:41:58,382 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=125, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-22T03:41:58,382 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=125, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-22T03:41:58,382 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=126, ppid=125, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-22T03:41:58,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-22T03:41:58,534 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42545 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-22T03:41:58,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795. 2024-11-22T03:41:58,535 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2902): Flushing 4a7ce0cfb17f4dcc3dd24b2156963795 3/3 column families, dataSize=161.02 KB heapSize=422.63 KB 2024-11-22T03:41:58,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/.tmp/A/cf322b78952c409f8183ae1743d6022a is 50, key is test_row_0/A:col10/1732246916428/Put/seqid=0 2024-11-22T03:41:58,544 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742247_1423 (size=12151) 2024-11-22T03:41:58,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(8855): Flush requested on 4a7ce0cfb17f4dcc3dd24b2156963795 2024-11-22T03:41:58,594 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795. as already flushing 2024-11-22T03:41:58,606 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:58,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40962 deadline: 1732246978598, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:58,606 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to c85114ed5096,42545,1732246786482 for 'test_row_1' in 4a7ce0cfb17f4dcc3dd24b2156963795 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2185 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-22T03:41:58,607 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:58,607 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:58,607 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:41:58,608 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:58,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40926 deadline: 1732246978603, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:58,609 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:58,609 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:58,609 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:41:58,613 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:58,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40990 deadline: 1732246978604, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:58,613 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:58,613 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:58,613 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:58,613 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:41:58,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40966 deadline: 1732246978604, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:58,614 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:58,614 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:58,614 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:41:58,622 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:58,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41022 deadline: 1732246978618, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:58,623 WARN [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRpcRetryingCaller(177): Call to c85114ed5096,42545,1732246786482 for 'test_row_2' in 4a7ce0cfb17f4dcc3dd24b2156963795 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2192 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-22T03:41:58,623 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:58,623 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:58,623 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:41:58,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-22T03:41:58,723 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:58,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40926 deadline: 1732246978718, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:58,723 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:58,723 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:58,723 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:58,723 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:41:58,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40990 deadline: 1732246978718, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:58,724 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:58,724 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:58,724 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:58,724 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:41:58,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40966 deadline: 1732246978718, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:58,724 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:58,724 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:58,724 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:41:58,933 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:58,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40926 deadline: 1732246978928, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:58,934 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:58,934 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:58,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40990 deadline: 1732246978928, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:58,934 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:58,934 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:41:58,934 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:58,934 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:58,934 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:58,934 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:41:58,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40966 deadline: 1732246978928, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:58,934 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:58,934 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:58,934 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:41:58,945 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=151 (bloomFilter=true), to=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/.tmp/A/cf322b78952c409f8183ae1743d6022a 2024-11-22T03:41:58,953 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/.tmp/B/3bb2b7f488f54c5595ce617df05a1b7d is 50, key is test_row_0/B:col10/1732246916428/Put/seqid=0 2024-11-22T03:41:58,962 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742248_1424 (size=12151) 2024-11-22T03:41:58,965 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=151 (bloomFilter=true), to=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/.tmp/B/3bb2b7f488f54c5595ce617df05a1b7d 2024-11-22T03:41:58,971 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/.tmp/C/79b8b9a3799c41b99c8bf3d1b12f38fa is 50, key is test_row_0/C:col10/1732246916428/Put/seqid=0 2024-11-22T03:41:58,979 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742249_1425 (size=12151) 2024-11-22T03:41:59,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-22T03:41:59,246 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:59,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40926 deadline: 1732246979239, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:59,246 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:59,246 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:59,246 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:59,246 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:41:59,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40990 deadline: 1732246979239, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:59,246 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:59,246 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:59,246 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:59,246 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:41:59,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40966 deadline: 1732246979239, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:59,247 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:59,247 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:59,247 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:41:59,379 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=151 (bloomFilter=true), to=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/.tmp/C/79b8b9a3799c41b99c8bf3d1b12f38fa 2024-11-22T03:41:59,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/.tmp/A/cf322b78952c409f8183ae1743d6022a as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/A/cf322b78952c409f8183ae1743d6022a 2024-11-22T03:41:59,391 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/A/cf322b78952c409f8183ae1743d6022a, entries=150, sequenceid=151, filesize=11.9 K 2024-11-22T03:41:59,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/.tmp/B/3bb2b7f488f54c5595ce617df05a1b7d as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/B/3bb2b7f488f54c5595ce617df05a1b7d 2024-11-22T03:41:59,395 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/B/3bb2b7f488f54c5595ce617df05a1b7d, entries=150, sequenceid=151, filesize=11.9 K 2024-11-22T03:41:59,396 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/.tmp/C/79b8b9a3799c41b99c8bf3d1b12f38fa as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/C/79b8b9a3799c41b99c8bf3d1b12f38fa 2024-11-22T03:41:59,399 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/C/79b8b9a3799c41b99c8bf3d1b12f38fa, entries=150, sequenceid=151, filesize=11.9 K 2024-11-22T03:41:59,400 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~422.58 KB/432720, currentSize=40.25 KB/41220 for 4a7ce0cfb17f4dcc3dd24b2156963795 in 865ms, sequenceid=151, compaction requested=true 2024-11-22T03:41:59,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2603): Flush status journal for 4a7ce0cfb17f4dcc3dd24b2156963795: 2024-11-22T03:41:59,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795. 2024-11-22T03:41:59,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=126 2024-11-22T03:41:59,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42627 {}] master.HMaster(4169): Remote procedure done, pid=126 2024-11-22T03:41:59,403 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=126, resume processing ppid=125 2024-11-22T03:41:59,403 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=126, ppid=125, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0190 sec 2024-11-22T03:41:59,404 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=125, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees in 1.0230 sec 2024-11-22T03:41:59,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-22T03:41:59,518 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-22T03:41:59,519 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42627 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-22T03:41:59,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42627 {}] procedure2.ProcedureExecutor(1139): Stored pid=127, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees 2024-11-22T03:41:59,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-22T03:41:59,520 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=127, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-22T03:41:59,520 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=127, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-22T03:41:59,521 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=128, ppid=127, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-22T03:41:59,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-22T03:41:59,672 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42545 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-22T03:41:59,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795. 2024-11-22T03:41:59,673 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2902): Flushing 4a7ce0cfb17f4dcc3dd24b2156963795 3/3 column families, dataSize=40.25 KB heapSize=106.22 KB 2024-11-22T03:41:59,675 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/.tmp/A/31722d9bf9d74620b78e43a09511105d is 50, key is test_row_0/A:col10/1732246918595/Put/seqid=0 2024-11-22T03:41:59,680 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742250_1426 (size=12151) 2024-11-22T03:41:59,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(8855): Flush requested on 4a7ce0cfb17f4dcc3dd24b2156963795 2024-11-22T03:41:59,762 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795. as already flushing 2024-11-22T03:41:59,838 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:59,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40926 deadline: 1732246979827, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:59,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-22T03:41:59,838 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:59,838 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:59,838 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:59,838 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:41:59,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40990 deadline: 1732246979829, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:59,838 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:59,838 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:59,838 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:41:59,841 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:59,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40966 deadline: 1732246979830, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:59,841 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:59,841 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:59,841 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:41:59,957 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:59,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40926 deadline: 1732246979949, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:59,958 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:59,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40990 deadline: 1732246979949, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:59,958 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:59,958 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:59,958 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:41:59,958 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:41:59,958 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:59,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40966 deadline: 1732246979949, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 2024-11-22T03:41:59,958 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:59,958 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:41:59,958 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:59,958 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:41:59,958 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:42:00,080 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=160 (bloomFilter=true), to=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/.tmp/A/31722d9bf9d74620b78e43a09511105d 2024-11-22T03:42:00,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/.tmp/B/554b916013a348b2b1739e3d0237de5f is 50, key is test_row_0/B:col10/1732246918595/Put/seqid=0 2024-11-22T03:42:00,089 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742251_1427 (size=12151) 2024-11-22T03:42:00,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-22T03:42:00,173 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:42:00,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40926 deadline: 1732246980169, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 2024-11-22T03:42:00,173 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:00,173 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:00,173 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:42:00,173 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:42:00,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40990 deadline: 1732246980169, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 2024-11-22T03:42:00,173 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:00,173 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:00,173 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:42:00,173 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:42:00,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40966 deadline: 1732246980170, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 2024-11-22T03:42:00,174 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:00,174 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:00,174 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:42:00,489 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:42:00,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40926 deadline: 1732246980478, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 2024-11-22T03:42:00,489 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:00,489 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:42:00,489 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:00,489 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:42:00,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40990 deadline: 1732246980478, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 2024-11-22T03:42:00,490 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:00,490 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:00,490 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:42:00,490 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:42:00,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40966 deadline: 1732246980479, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 2024-11-22T03:42:00,490 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:00,490 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:00,490 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:42:00,491 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=160 (bloomFilter=true), to=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/.tmp/B/554b916013a348b2b1739e3d0237de5f 2024-11-22T03:42:00,496 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/.tmp/C/286c073c0a3d466e86b38934780fbd46 is 50, key is test_row_0/C:col10/1732246918595/Put/seqid=0 2024-11-22T03:42:00,498 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742252_1428 (size=12151) 2024-11-22T03:42:00,498 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=160 (bloomFilter=true), to=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/.tmp/C/286c073c0a3d466e86b38934780fbd46 2024-11-22T03:42:00,503 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/.tmp/A/31722d9bf9d74620b78e43a09511105d as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/A/31722d9bf9d74620b78e43a09511105d 2024-11-22T03:42:00,508 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/A/31722d9bf9d74620b78e43a09511105d, entries=150, sequenceid=160, filesize=11.9 K 2024-11-22T03:42:00,508 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/.tmp/B/554b916013a348b2b1739e3d0237de5f as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/B/554b916013a348b2b1739e3d0237de5f 2024-11-22T03:42:00,511 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/B/554b916013a348b2b1739e3d0237de5f, entries=150, sequenceid=160, filesize=11.9 K 2024-11-22T03:42:00,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/.tmp/C/286c073c0a3d466e86b38934780fbd46 as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/C/286c073c0a3d466e86b38934780fbd46 2024-11-22T03:42:00,514 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/C/286c073c0a3d466e86b38934780fbd46, entries=150, sequenceid=160, filesize=11.9 K 2024-11-22T03:42:00,514 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~106.17 KB/108720, currentSize=161.02 KB/164880 for 4a7ce0cfb17f4dcc3dd24b2156963795 in 842ms, sequenceid=160, compaction requested=true 2024-11-22T03:42:00,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2603): Flush status journal for 4a7ce0cfb17f4dcc3dd24b2156963795: 2024-11-22T03:42:00,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795. 2024-11-22T03:42:00,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=128 2024-11-22T03:42:00,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42627 {}] master.HMaster(4169): Remote procedure done, pid=128 2024-11-22T03:42:00,517 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=128, resume processing ppid=127 2024-11-22T03:42:00,517 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=128, ppid=127, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 994 msec 2024-11-22T03:42:00,518 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=127, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees in 998 msec 2024-11-22T03:42:00,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(8855): Flush requested on 4a7ce0cfb17f4dcc3dd24b2156963795 2024-11-22T03:42:00,628 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-22T03:42:00,628 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4a7ce0cfb17f4dcc3dd24b2156963795 3/3 column families, dataSize=167.72 KB heapSize=440.20 KB 2024-11-22T03:42:00,633 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/.tmp/A/bc05629301bf427595f60e80b6b07a2e is 50, key is test_row_0/A:col10/1732246919817/Put/seqid=0 2024-11-22T03:42:00,636 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742253_1429 (size=14541) 2024-11-22T03:42:00,656 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:42:00,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40962 deadline: 1732246980655, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 2024-11-22T03:42:00,657 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:00,657 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:00,657 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:42:00,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-22T03:42:00,659 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-22T03:42:00,659 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42627 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-22T03:42:00,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42627 {}] procedure2.ProcedureExecutor(1139): Stored pid=129, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees 2024-11-22T03:42:00,660 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=129, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-22T03:42:00,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-22T03:42:00,661 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=129, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-22T03:42:00,661 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=130, ppid=129, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-22T03:42:00,668 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:42:00,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41022 deadline: 1732246980656, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 2024-11-22T03:42:00,669 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:00,669 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:00,669 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:42:00,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-22T03:42:00,770 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:42:00,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40962 deadline: 1732246980769, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 2024-11-22T03:42:00,772 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:00,772 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:00,772 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:42:00,785 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:42:00,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41022 deadline: 1732246980778, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 2024-11-22T03:42:00,786 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:00,786 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:00,786 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:42:00,813 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42545 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-22T03:42:00,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795. 2024-11-22T03:42:00,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795. as already flushing 2024-11-22T03:42:00,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795. 2024-11-22T03:42:00,813 ERROR [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 4a7ce0cfb17f4dcc3dd24b2156963795, NAME => 'TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:42:00,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 4a7ce0cfb17f4dcc3dd24b2156963795, NAME => 'TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:42:00,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42627 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4a7ce0cfb17f4dcc3dd24b2156963795, NAME => 'TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4a7ce0cfb17f4dcc3dd24b2156963795, NAME => 'TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:42:00,965 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42545 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-22T03:42:00,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795. 2024-11-22T03:42:00,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795. as already flushing 2024-11-22T03:42:00,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795. 2024-11-22T03:42:00,965 ERROR [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 4a7ce0cfb17f4dcc3dd24b2156963795, NAME => 'TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:42:00,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 4a7ce0cfb17f4dcc3dd24b2156963795, NAME => 'TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:42:00,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42627 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4a7ce0cfb17f4dcc3dd24b2156963795, NAME => 'TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4a7ce0cfb17f4dcc3dd24b2156963795, NAME => 'TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:42:00,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-22T03:42:00,986 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:42:00,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40962 deadline: 1732246980979, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 2024-11-22T03:42:00,986 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:00,986 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:00,986 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:42:01,000 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:42:01,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41022 deadline: 1732246980988, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 2024-11-22T03:42:01,001 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:01,001 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:01,001 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:42:01,004 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:42:01,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40926 deadline: 1732246980998, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 2024-11-22T03:42:01,005 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:01,005 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:42:01,005 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:01,005 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:42:01,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40990 deadline: 1732246980998, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 2024-11-22T03:42:01,005 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:01,005 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:01,005 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:42:01,005 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:42:01,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40966 deadline: 1732246980999, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 2024-11-22T03:42:01,005 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:01,005 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:01,005 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:42:01,036 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=188 (bloomFilter=true), to=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/.tmp/A/bc05629301bf427595f60e80b6b07a2e 2024-11-22T03:42:01,042 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/.tmp/B/6b664097f71749ed85806a4b6afcb230 is 50, key is test_row_0/B:col10/1732246919817/Put/seqid=0 2024-11-22T03:42:01,050 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742254_1430 (size=12151) 2024-11-22T03:42:01,118 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42545 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-22T03:42:01,118 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795. 2024-11-22T03:42:01,118 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795. as already flushing 2024-11-22T03:42:01,118 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795. 2024-11-22T03:42:01,118 ERROR [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 4a7ce0cfb17f4dcc3dd24b2156963795, NAME => 'TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:42:01,119 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 4a7ce0cfb17f4dcc3dd24b2156963795, NAME => 'TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:42:01,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42627 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4a7ce0cfb17f4dcc3dd24b2156963795, NAME => 'TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4a7ce0cfb17f4dcc3dd24b2156963795, NAME => 'TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:42:01,271 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42545 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-22T03:42:01,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795. 2024-11-22T03:42:01,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795. as already flushing 2024-11-22T03:42:01,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795. 2024-11-22T03:42:01,271 ERROR [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 4a7ce0cfb17f4dcc3dd24b2156963795, NAME => 'TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:42:01,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 4a7ce0cfb17f4dcc3dd24b2156963795, NAME => 'TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:42:01,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42627 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4a7ce0cfb17f4dcc3dd24b2156963795, NAME => 'TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4a7ce0cfb17f4dcc3dd24b2156963795, NAME => 'TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:42:01,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-22T03:42:01,306 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:42:01,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40962 deadline: 1732246981298, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 2024-11-22T03:42:01,306 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:01,306 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:01,306 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:42:01,316 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:42:01,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41022 deadline: 1732246981308, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 2024-11-22T03:42:01,316 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:01,316 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:01,316 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:42:01,424 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42545 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-22T03:42:01,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795. 2024-11-22T03:42:01,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795. as already flushing 2024-11-22T03:42:01,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795. 2024-11-22T03:42:01,424 ERROR [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 4a7ce0cfb17f4dcc3dd24b2156963795, NAME => 'TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:42:01,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 4a7ce0cfb17f4dcc3dd24b2156963795, NAME => 'TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:42:01,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42627 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4a7ce0cfb17f4dcc3dd24b2156963795, NAME => 'TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4a7ce0cfb17f4dcc3dd24b2156963795, NAME => 'TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:42:01,450 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=188 (bloomFilter=true), to=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/.tmp/B/6b664097f71749ed85806a4b6afcb230 2024-11-22T03:42:01,456 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/.tmp/C/9cfaacff5cd64b3594653beaf1dca79e is 50, key is test_row_0/C:col10/1732246919817/Put/seqid=0 2024-11-22T03:42:01,464 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742255_1431 (size=12151) 2024-11-22T03:42:01,576 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42545 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-22T03:42:01,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795. 2024-11-22T03:42:01,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795. as already flushing 2024-11-22T03:42:01,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795. 2024-11-22T03:42:01,576 ERROR [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 4a7ce0cfb17f4dcc3dd24b2156963795, NAME => 'TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:42:01,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 4a7ce0cfb17f4dcc3dd24b2156963795, NAME => 'TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:42:01,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42627 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4a7ce0cfb17f4dcc3dd24b2156963795, NAME => 'TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4a7ce0cfb17f4dcc3dd24b2156963795, NAME => 'TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:42:01,729 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42545 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-22T03:42:01,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795. 2024-11-22T03:42:01,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795. as already flushing 2024-11-22T03:42:01,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795. 2024-11-22T03:42:01,729 ERROR [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 4a7ce0cfb17f4dcc3dd24b2156963795, NAME => 'TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:42:01,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 4a7ce0cfb17f4dcc3dd24b2156963795, NAME => 'TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:42:01,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42627 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4a7ce0cfb17f4dcc3dd24b2156963795, NAME => 'TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4a7ce0cfb17f4dcc3dd24b2156963795, NAME => 'TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:42:01,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-22T03:42:01,822 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:42:01,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40962 deadline: 1732246981818, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 2024-11-22T03:42:01,823 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:01,823 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:01,823 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:42:01,832 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:42:01,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41022 deadline: 1732246981829, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 2024-11-22T03:42:01,832 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:01,832 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:01,832 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:42:01,865 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=188 (bloomFilter=true), to=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/.tmp/C/9cfaacff5cd64b3594653beaf1dca79e 2024-11-22T03:42:01,868 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/.tmp/A/bc05629301bf427595f60e80b6b07a2e as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/A/bc05629301bf427595f60e80b6b07a2e 2024-11-22T03:42:01,870 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/A/bc05629301bf427595f60e80b6b07a2e, entries=200, sequenceid=188, filesize=14.2 K 2024-11-22T03:42:01,871 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/.tmp/B/6b664097f71749ed85806a4b6afcb230 as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/B/6b664097f71749ed85806a4b6afcb230 2024-11-22T03:42:01,873 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/B/6b664097f71749ed85806a4b6afcb230, entries=150, sequenceid=188, filesize=11.9 K 2024-11-22T03:42:01,874 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/.tmp/C/9cfaacff5cd64b3594653beaf1dca79e as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/C/9cfaacff5cd64b3594653beaf1dca79e 2024-11-22T03:42:01,876 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/C/9cfaacff5cd64b3594653beaf1dca79e, entries=150, sequenceid=188, filesize=11.9 K 2024-11-22T03:42:01,877 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~167.72 KB/171750, heapSize ~440.16 KB/450720, currentSize=33.54 KB/34350 for 4a7ce0cfb17f4dcc3dd24b2156963795 in 1249ms, sequenceid=188, compaction requested=true 2024-11-22T03:42:01,877 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4a7ce0cfb17f4dcc3dd24b2156963795: 2024-11-22T03:42:01,877 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4a7ce0cfb17f4dcc3dd24b2156963795:A, priority=-2147483648, current under compaction store size is 1 2024-11-22T03:42:01,877 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T03:42:01,877 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 6 store files, 0 compacting, 6 eligible, 16 blocking 2024-11-22T03:42:01,877 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4a7ce0cfb17f4dcc3dd24b2156963795:B, priority=-2147483648, current under compaction store size is 2 2024-11-22T03:42:01,877 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T03:42:01,877 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 6 store files, 0 compacting, 6 eligible, 16 blocking 2024-11-22T03:42:01,877 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4a7ce0cfb17f4dcc3dd24b2156963795:C, priority=-2147483648, current under compaction store size is 3 2024-11-22T03:42:01,877 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T03:42:01,878 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 6 files of size 79732 starting at candidate #0 after considering 10 permutations with 10 in ratio 2024-11-22T03:42:01,878 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 6 files of size 72662 starting at candidate #0 after considering 10 permutations with 10 in ratio 2024-11-22T03:42:01,878 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HStore(1541): 4a7ce0cfb17f4dcc3dd24b2156963795/B is initiating minor compaction (all files) 2024-11-22T03:42:01,878 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.HStore(1541): 4a7ce0cfb17f4dcc3dd24b2156963795/A is initiating minor compaction (all files) 2024-11-22T03:42:01,878 INFO [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4a7ce0cfb17f4dcc3dd24b2156963795/B in TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795. 2024-11-22T03:42:01,878 INFO [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4a7ce0cfb17f4dcc3dd24b2156963795/A in TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795. 2024-11-22T03:42:01,878 INFO [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/B/ac98440d556545e0ae9f8aa814757c9d, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/B/9390ed4698b340348b6f6a8156f9b8da, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/B/26dc1f560f78442da4005f4162884cfe, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/B/3bb2b7f488f54c5595ce617df05a1b7d, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/B/554b916013a348b2b1739e3d0237de5f, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/B/6b664097f71749ed85806a4b6afcb230] into tmpdir=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/.tmp, totalSize=71.0 K 2024-11-22T03:42:01,878 INFO [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/A/94b11a7be9aa448885581b203cbb3c99, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/A/8d1d6eaa02cc4493ac2e1c802f395cb4, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/A/7310966b824440a9bb996c72dc5b5e1a, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/A/cf322b78952c409f8183ae1743d6022a, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/A/31722d9bf9d74620b78e43a09511105d, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/A/bc05629301bf427595f60e80b6b07a2e] into tmpdir=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/.tmp, totalSize=77.9 K 2024-11-22T03:42:01,879 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] compactions.Compactor(225): Compacting 94b11a7be9aa448885581b203cbb3c99, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=91, earliestPutTs=1732246913850 2024-11-22T03:42:01,879 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] compactions.Compactor(225): Compacting ac98440d556545e0ae9f8aa814757c9d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=91, earliestPutTs=1732246913850 2024-11-22T03:42:01,879 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] compactions.Compactor(225): Compacting 9390ed4698b340348b6f6a8156f9b8da, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=113, earliestPutTs=1732246914199 2024-11-22T03:42:01,879 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8d1d6eaa02cc4493ac2e1c802f395cb4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=113, earliestPutTs=1732246914199 2024-11-22T03:42:01,879 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] compactions.Compactor(225): Compacting 26dc1f560f78442da4005f4162884cfe, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=124, earliestPutTs=1732246916380 2024-11-22T03:42:01,879 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7310966b824440a9bb996c72dc5b5e1a, keycount=250, bloomtype=ROW, size=16.3 K, encoding=NONE, compression=NONE, seqNum=124, earliestPutTs=1732246916380 2024-11-22T03:42:01,879 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] compactions.Compactor(225): Compacting cf322b78952c409f8183ae1743d6022a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=151, earliestPutTs=1732246916422 2024-11-22T03:42:01,879 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] compactions.Compactor(225): Compacting 3bb2b7f488f54c5595ce617df05a1b7d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=151, earliestPutTs=1732246916422 2024-11-22T03:42:01,879 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] compactions.Compactor(225): Compacting 554b916013a348b2b1739e3d0237de5f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=160, earliestPutTs=1732246918595 2024-11-22T03:42:01,879 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] compactions.Compactor(225): Compacting 31722d9bf9d74620b78e43a09511105d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=160, earliestPutTs=1732246918595 2024-11-22T03:42:01,880 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] compactions.Compactor(225): Compacting 6b664097f71749ed85806a4b6afcb230, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=188, earliestPutTs=1732246919817 2024-11-22T03:42:01,880 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] compactions.Compactor(225): Compacting bc05629301bf427595f60e80b6b07a2e, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=188, earliestPutTs=1732246919817 2024-11-22T03:42:01,881 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42545 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-22T03:42:01,881 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795. 2024-11-22T03:42:01,882 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2902): Flushing 4a7ce0cfb17f4dcc3dd24b2156963795 3/3 column families, dataSize=33.54 KB heapSize=88.64 KB 2024-11-22T03:42:01,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/.tmp/A/8e923d4205144abea3d6f5e110f7961b is 50, key is test_row_0/A:col10/1732246920655/Put/seqid=0 2024-11-22T03:42:01,890 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742256_1432 (size=12151) 2024-11-22T03:42:01,891 INFO [RS:0;c85114ed5096:42545-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4a7ce0cfb17f4dcc3dd24b2156963795#B#compaction#367 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T03:42:01,891 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/.tmp/B/031726fec90d4c01ae917ca441ec14d5 is 50, key is test_row_0/B:col10/1732246919817/Put/seqid=0 2024-11-22T03:42:01,892 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=196 (bloomFilter=true), to=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/.tmp/A/8e923d4205144abea3d6f5e110f7961b 2024-11-22T03:42:01,902 INFO [RS:0;c85114ed5096:42545-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4a7ce0cfb17f4dcc3dd24b2156963795#A#compaction#368 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T03:42:01,902 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/.tmp/A/892770ca81db42de8adbb8f81e60a526 is 50, key is test_row_0/A:col10/1732246919817/Put/seqid=0 2024-11-22T03:42:01,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/.tmp/B/96ff445498f445dc8dcd512fb3da5c27 is 50, key is test_row_0/B:col10/1732246920655/Put/seqid=0 2024-11-22T03:42:01,910 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742257_1433 (size=12561) 2024-11-22T03:42:01,918 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742258_1434 (size=12561) 2024-11-22T03:42:01,918 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742259_1435 (size=12151) 2024-11-22T03:42:01,922 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/.tmp/A/892770ca81db42de8adbb8f81e60a526 as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/A/892770ca81db42de8adbb8f81e60a526 2024-11-22T03:42:01,926 INFO [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 6 (all) file(s) in 4a7ce0cfb17f4dcc3dd24b2156963795/A of 4a7ce0cfb17f4dcc3dd24b2156963795 into 892770ca81db42de8adbb8f81e60a526(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T03:42:01,926 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4a7ce0cfb17f4dcc3dd24b2156963795: 2024-11-22T03:42:01,926 INFO [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., storeName=4a7ce0cfb17f4dcc3dd24b2156963795/A, priority=10, startTime=1732246921877; duration=0sec 2024-11-22T03:42:01,926 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T03:42:01,926 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4a7ce0cfb17f4dcc3dd24b2156963795:A 2024-11-22T03:42:01,926 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 6 store files, 0 compacting, 6 eligible, 16 blocking 2024-11-22T03:42:01,927 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 6 files of size 72662 starting at candidate #0 after considering 10 permutations with 10 in ratio 2024-11-22T03:42:01,927 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.HStore(1541): 4a7ce0cfb17f4dcc3dd24b2156963795/C is initiating minor compaction (all files) 2024-11-22T03:42:01,927 INFO [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4a7ce0cfb17f4dcc3dd24b2156963795/C in TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795. 2024-11-22T03:42:01,927 INFO [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/C/e897aad97196496696b4d82b2f2b5e80, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/C/e79f25f1f6a04dde8e1c6fe870de6db4, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/C/40a6b580d41747a581790e5e89fa6652, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/C/79b8b9a3799c41b99c8bf3d1b12f38fa, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/C/286c073c0a3d466e86b38934780fbd46, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/C/9cfaacff5cd64b3594653beaf1dca79e] into tmpdir=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/.tmp, totalSize=71.0 K 2024-11-22T03:42:01,927 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] compactions.Compactor(225): Compacting e897aad97196496696b4d82b2f2b5e80, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=91, earliestPutTs=1732246913850 2024-11-22T03:42:01,927 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] compactions.Compactor(225): Compacting e79f25f1f6a04dde8e1c6fe870de6db4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=113, earliestPutTs=1732246914199 2024-11-22T03:42:01,929 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] compactions.Compactor(225): Compacting 40a6b580d41747a581790e5e89fa6652, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=124, earliestPutTs=1732246916380 2024-11-22T03:42:01,929 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] compactions.Compactor(225): Compacting 79b8b9a3799c41b99c8bf3d1b12f38fa, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=151, earliestPutTs=1732246916422 2024-11-22T03:42:01,929 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] compactions.Compactor(225): Compacting 286c073c0a3d466e86b38934780fbd46, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=160, earliestPutTs=1732246918595 2024-11-22T03:42:01,929 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9cfaacff5cd64b3594653beaf1dca79e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=188, earliestPutTs=1732246919817 2024-11-22T03:42:01,939 INFO [RS:0;c85114ed5096:42545-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4a7ce0cfb17f4dcc3dd24b2156963795#C#compaction#370 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T03:42:01,939 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/.tmp/C/bc04f226748a48668bb72884ac2ba244 is 50, key is test_row_0/C:col10/1732246919817/Put/seqid=0 2024-11-22T03:42:01,944 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742260_1436 (size=12561) 2024-11-22T03:42:01,952 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/.tmp/C/bc04f226748a48668bb72884ac2ba244 as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/C/bc04f226748a48668bb72884ac2ba244 2024-11-22T03:42:01,961 INFO [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 6 (all) file(s) in 4a7ce0cfb17f4dcc3dd24b2156963795/C of 4a7ce0cfb17f4dcc3dd24b2156963795 into bc04f226748a48668bb72884ac2ba244(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T03:42:01,961 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4a7ce0cfb17f4dcc3dd24b2156963795: 2024-11-22T03:42:01,961 INFO [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., storeName=4a7ce0cfb17f4dcc3dd24b2156963795/C, priority=10, startTime=1732246921877; duration=0sec 2024-11-22T03:42:01,961 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T03:42:01,961 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4a7ce0cfb17f4dcc3dd24b2156963795:C 2024-11-22T03:42:02,018 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795. as already flushing 2024-11-22T03:42:02,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(8855): Flush requested on 4a7ce0cfb17f4dcc3dd24b2156963795 2024-11-22T03:42:02,084 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:42:02,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40990 deadline: 1732246982073, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 2024-11-22T03:42:02,084 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:02,084 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:02,084 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:42:02,091 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:42:02,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40926 deadline: 1732246982083, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 2024-11-22T03:42:02,091 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:02,091 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:02,091 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:42:02,092 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:42:02,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40966 deadline: 1732246982084, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 2024-11-22T03:42:02,092 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:02,092 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:02,092 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:42:02,195 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:42:02,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40990 deadline: 1732246982189, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 2024-11-22T03:42:02,195 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:02,195 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:02,195 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:42:02,205 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:42:02,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40926 deadline: 1732246982199, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 2024-11-22T03:42:02,206 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:02,206 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:42:02,206 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:02,206 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:42:02,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40966 deadline: 1732246982199, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 2024-11-22T03:42:02,206 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:02,206 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:02,206 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:42:02,314 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/.tmp/B/031726fec90d4c01ae917ca441ec14d5 as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/B/031726fec90d4c01ae917ca441ec14d5 2024-11-22T03:42:02,317 INFO [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 6 (all) file(s) in 4a7ce0cfb17f4dcc3dd24b2156963795/B of 4a7ce0cfb17f4dcc3dd24b2156963795 into 031726fec90d4c01ae917ca441ec14d5(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T03:42:02,317 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4a7ce0cfb17f4dcc3dd24b2156963795: 2024-11-22T03:42:02,318 INFO [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., storeName=4a7ce0cfb17f4dcc3dd24b2156963795/B, priority=10, startTime=1732246921877; duration=0sec 2024-11-22T03:42:02,318 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T03:42:02,318 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4a7ce0cfb17f4dcc3dd24b2156963795:B 2024-11-22T03:42:02,318 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=196 (bloomFilter=true), to=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/.tmp/B/96ff445498f445dc8dcd512fb3da5c27 2024-11-22T03:42:02,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/.tmp/C/a7152235b6094b9c8abcb0cf1c779870 is 50, key is test_row_0/C:col10/1732246920655/Put/seqid=0 2024-11-22T03:42:02,326 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742261_1437 (size=12151) 2024-11-22T03:42:02,403 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:42:02,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40990 deadline: 1732246982398, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 2024-11-22T03:42:02,403 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:02,403 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:02,403 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:42:02,412 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:42:02,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40926 deadline: 1732246982409, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 2024-11-22T03:42:02,412 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:42:02,412 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:02,412 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:02,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40966 deadline: 1732246982409, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 2024-11-22T03:42:02,412 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:42:02,412 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:02,412 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:02,412 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:42:02,710 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:42:02,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40990 deadline: 1732246982708, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 2024-11-22T03:42:02,711 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:02,711 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:02,711 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:42:02,721 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:42:02,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40926 deadline: 1732246982719, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 2024-11-22T03:42:02,722 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:42:02,722 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:02,722 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:02,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40966 deadline: 1732246982719, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 2024-11-22T03:42:02,722 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:42:02,722 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:02,722 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:02,722 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:42:02,726 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=196 (bloomFilter=true), to=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/.tmp/C/a7152235b6094b9c8abcb0cf1c779870 2024-11-22T03:42:02,730 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/.tmp/A/8e923d4205144abea3d6f5e110f7961b as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/A/8e923d4205144abea3d6f5e110f7961b 2024-11-22T03:42:02,736 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/A/8e923d4205144abea3d6f5e110f7961b, entries=150, sequenceid=196, filesize=11.9 K 2024-11-22T03:42:02,737 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/.tmp/B/96ff445498f445dc8dcd512fb3da5c27 as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/B/96ff445498f445dc8dcd512fb3da5c27 2024-11-22T03:42:02,743 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/B/96ff445498f445dc8dcd512fb3da5c27, entries=150, sequenceid=196, filesize=11.9 K 2024-11-22T03:42:02,744 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/.tmp/C/a7152235b6094b9c8abcb0cf1c779870 as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/C/a7152235b6094b9c8abcb0cf1c779870 2024-11-22T03:42:02,748 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/C/a7152235b6094b9c8abcb0cf1c779870, entries=150, sequenceid=196, filesize=11.9 K 2024-11-22T03:42:02,749 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=167.72 KB/171750 for 4a7ce0cfb17f4dcc3dd24b2156963795 in 868ms, sequenceid=196, compaction requested=false 2024-11-22T03:42:02,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2603): Flush status journal for 4a7ce0cfb17f4dcc3dd24b2156963795: 2024-11-22T03:42:02,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795. 2024-11-22T03:42:02,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=130 2024-11-22T03:42:02,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42627 {}] master.HMaster(4169): Remote procedure done, pid=130 2024-11-22T03:42:02,752 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=130, resume processing ppid=129 2024-11-22T03:42:02,753 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=130, ppid=129, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0890 sec 2024-11-22T03:42:02,754 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=129, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees in 2.0940 sec 2024-11-22T03:42:02,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-22T03:42:02,809 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-22T03:42:02,809 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42627 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-22T03:42:02,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42627 {}] procedure2.ProcedureExecutor(1139): Stored pid=131, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees 2024-11-22T03:42:02,811 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=131, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-22T03:42:02,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-22T03:42:02,812 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=131, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-22T03:42:02,812 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=132, ppid=131, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-22T03:42:02,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(8855): Flush requested on 4a7ce0cfb17f4dcc3dd24b2156963795 2024-11-22T03:42:02,835 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-22T03:42:02,835 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4a7ce0cfb17f4dcc3dd24b2156963795 3/3 column families, dataSize=174.43 KB heapSize=457.78 KB 2024-11-22T03:42:02,839 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/.tmp/A/44ab2d6996b046d493a4e6f63b3d09cd is 50, key is test_row_0/A:col10/1732246922061/Put/seqid=0 2024-11-22T03:42:02,845 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742262_1438 (size=14541) 2024-11-22T03:42:02,853 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:42:02,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40962 deadline: 1732246982849, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 2024-11-22T03:42:02,853 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:02,854 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:02,854 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:42:02,856 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:42:02,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41022 deadline: 1732246982851, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 2024-11-22T03:42:02,860 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:02,860 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:02,860 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:42:02,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-22T03:42:02,964 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42545 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-22T03:42:02,964 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795. 2024-11-22T03:42:02,964 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795. as already flushing 2024-11-22T03:42:02,964 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795. 2024-11-22T03:42:02,964 ERROR [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 4a7ce0cfb17f4dcc3dd24b2156963795, NAME => 'TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:42:02,964 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 4a7ce0cfb17f4dcc3dd24b2156963795, NAME => 'TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:42:02,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42627 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4a7ce0cfb17f4dcc3dd24b2156963795, NAME => 'TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4a7ce0cfb17f4dcc3dd24b2156963795, NAME => 'TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:42:02,965 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:42:02,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40962 deadline: 1732246982958, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 2024-11-22T03:42:02,966 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:02,966 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:02,966 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:42:02,975 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:42:02,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41022 deadline: 1732246982968, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 2024-11-22T03:42:02,975 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:02,975 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:02,975 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:42:03,117 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42545 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-22T03:42:03,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795. 2024-11-22T03:42:03,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795. as already flushing 2024-11-22T03:42:03,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795. 2024-11-22T03:42:03,117 ERROR [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 4a7ce0cfb17f4dcc3dd24b2156963795, NAME => 'TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:42:03,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 4a7ce0cfb17f4dcc3dd24b2156963795, NAME => 'TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:42:03,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42627 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4a7ce0cfb17f4dcc3dd24b2156963795, NAME => 'TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4a7ce0cfb17f4dcc3dd24b2156963795, NAME => 'TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:42:03,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-22T03:42:03,172 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:42:03,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40962 deadline: 1732246983168, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 2024-11-22T03:42:03,172 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:03,172 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:03,172 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:42:03,179 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:42:03,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41022 deadline: 1732246983179, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 2024-11-22T03:42:03,180 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:03,180 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:03,180 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:42:03,221 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:42:03,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40990 deadline: 1732246983218, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 2024-11-22T03:42:03,222 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:03,222 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:03,222 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:42:03,231 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:42:03,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40926 deadline: 1732246983228, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 2024-11-22T03:42:03,231 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:42:03,231 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:03,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40966 deadline: 1732246983229, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 2024-11-22T03:42:03,231 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:03,231 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:42:03,231 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:03,231 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:03,231 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:42:03,246 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58.14 KB at sequenceid=228 (bloomFilter=true), to=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/.tmp/A/44ab2d6996b046d493a4e6f63b3d09cd 2024-11-22T03:42:03,259 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/.tmp/B/9bcaa526e82641cab5ce90d52f648996 is 50, key is test_row_0/B:col10/1732246922061/Put/seqid=0 2024-11-22T03:42:03,267 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742263_1439 (size=12151) 2024-11-22T03:42:03,269 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42545 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-22T03:42:03,269 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795. 2024-11-22T03:42:03,269 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795. as already flushing 2024-11-22T03:42:03,269 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795. 2024-11-22T03:42:03,269 ERROR [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 4a7ce0cfb17f4dcc3dd24b2156963795, NAME => 'TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:42:03,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 4a7ce0cfb17f4dcc3dd24b2156963795, NAME => 'TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:42:03,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42627 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4a7ce0cfb17f4dcc3dd24b2156963795, NAME => 'TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4a7ce0cfb17f4dcc3dd24b2156963795, NAME => 'TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:42:03,422 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42545 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-22T03:42:03,422 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795. 2024-11-22T03:42:03,422 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795. as already flushing 2024-11-22T03:42:03,422 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795. 2024-11-22T03:42:03,422 ERROR [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 4a7ce0cfb17f4dcc3dd24b2156963795, NAME => 'TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:42:03,422 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 4a7ce0cfb17f4dcc3dd24b2156963795, NAME => 'TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:42:03,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42627 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4a7ce0cfb17f4dcc3dd24b2156963795, NAME => 'TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4a7ce0cfb17f4dcc3dd24b2156963795, NAME => 'TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:42:03,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-22T03:42:03,482 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:42:03,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40962 deadline: 1732246983478, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 2024-11-22T03:42:03,483 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:03,483 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:03,483 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:42:03,492 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:42:03,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41022 deadline: 1732246983488, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 2024-11-22T03:42:03,492 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:03,492 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:03,493 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:42:03,574 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42545 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-22T03:42:03,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795. 2024-11-22T03:42:03,575 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795. as already flushing 2024-11-22T03:42:03,575 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795. 2024-11-22T03:42:03,575 ERROR [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 4a7ce0cfb17f4dcc3dd24b2156963795, NAME => 'TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:42:03,575 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 4a7ce0cfb17f4dcc3dd24b2156963795, NAME => 'TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:42:03,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42627 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4a7ce0cfb17f4dcc3dd24b2156963795, NAME => 'TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4a7ce0cfb17f4dcc3dd24b2156963795, NAME => 'TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:42:03,667 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58.14 KB at sequenceid=228 (bloomFilter=true), to=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/.tmp/B/9bcaa526e82641cab5ce90d52f648996 2024-11-22T03:42:03,674 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/.tmp/C/3145934cf1ca4770afed9477b8de31de is 50, key is test_row_0/C:col10/1732246922061/Put/seqid=0 2024-11-22T03:42:03,681 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742264_1440 (size=12151) 2024-11-22T03:42:03,727 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42545 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-22T03:42:03,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795. 2024-11-22T03:42:03,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795. as already flushing 2024-11-22T03:42:03,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795. 2024-11-22T03:42:03,727 ERROR [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 4a7ce0cfb17f4dcc3dd24b2156963795, NAME => 'TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:42:03,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 4a7ce0cfb17f4dcc3dd24b2156963795, NAME => 'TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:42:03,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42627 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4a7ce0cfb17f4dcc3dd24b2156963795, NAME => 'TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4a7ce0cfb17f4dcc3dd24b2156963795, NAME => 'TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:42:03,880 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42545 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-22T03:42:03,880 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795. 2024-11-22T03:42:03,880 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795. as already flushing 2024-11-22T03:42:03,880 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795. 2024-11-22T03:42:03,880 ERROR [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 4a7ce0cfb17f4dcc3dd24b2156963795, NAME => 'TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:42:03,880 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 4a7ce0cfb17f4dcc3dd24b2156963795, NAME => 'TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:42:03,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42627 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4a7ce0cfb17f4dcc3dd24b2156963795, NAME => 'TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4a7ce0cfb17f4dcc3dd24b2156963795, NAME => 'TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:42:03,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-22T03:42:03,995 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:42:03,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40962 deadline: 1732246983988, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 2024-11-22T03:42:03,996 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:03,996 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:03,996 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:42:04,005 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:42:04,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41022 deadline: 1732246983998, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 2024-11-22T03:42:04,005 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:04,005 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:04,005 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:42:04,032 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42545 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-22T03:42:04,032 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795. 2024-11-22T03:42:04,032 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795. as already flushing 2024-11-22T03:42:04,033 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795. 2024-11-22T03:42:04,033 ERROR [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 4a7ce0cfb17f4dcc3dd24b2156963795, NAME => 'TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:42:04,033 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 4a7ce0cfb17f4dcc3dd24b2156963795, NAME => 'TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:42:04,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42627 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4a7ce0cfb17f4dcc3dd24b2156963795, NAME => 'TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4a7ce0cfb17f4dcc3dd24b2156963795, NAME => 'TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:42:04,082 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58.14 KB at sequenceid=228 (bloomFilter=true), to=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/.tmp/C/3145934cf1ca4770afed9477b8de31de 2024-11-22T03:42:04,085 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/.tmp/A/44ab2d6996b046d493a4e6f63b3d09cd as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/A/44ab2d6996b046d493a4e6f63b3d09cd 2024-11-22T03:42:04,088 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/A/44ab2d6996b046d493a4e6f63b3d09cd, entries=200, sequenceid=228, filesize=14.2 K 2024-11-22T03:42:04,089 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/.tmp/B/9bcaa526e82641cab5ce90d52f648996 as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/B/9bcaa526e82641cab5ce90d52f648996 2024-11-22T03:42:04,091 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/B/9bcaa526e82641cab5ce90d52f648996, entries=150, sequenceid=228, filesize=11.9 K 2024-11-22T03:42:04,091 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/.tmp/C/3145934cf1ca4770afed9477b8de31de as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/C/3145934cf1ca4770afed9477b8de31de 2024-11-22T03:42:04,094 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/C/3145934cf1ca4770afed9477b8de31de, entries=150, sequenceid=228, filesize=11.9 K 2024-11-22T03:42:04,094 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~174.43 KB/178620, heapSize ~457.73 KB/468720, currentSize=26.84 KB/27480 for 4a7ce0cfb17f4dcc3dd24b2156963795 in 1259ms, sequenceid=228, compaction requested=true 2024-11-22T03:42:04,095 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4a7ce0cfb17f4dcc3dd24b2156963795: 2024-11-22T03:42:04,095 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4a7ce0cfb17f4dcc3dd24b2156963795:A, priority=-2147483648, current under compaction store size is 1 2024-11-22T03:42:04,095 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T03:42:04,095 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T03:42:04,095 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T03:42:04,095 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4a7ce0cfb17f4dcc3dd24b2156963795:B, priority=-2147483648, current under compaction store size is 2 2024-11-22T03:42:04,095 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T03:42:04,095 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4a7ce0cfb17f4dcc3dd24b2156963795:C, priority=-2147483648, current under compaction store size is 3 2024-11-22T03:42:04,095 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T03:42:04,095 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39253 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T03:42:04,095 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T03:42:04,095 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.HStore(1541): 4a7ce0cfb17f4dcc3dd24b2156963795/A is initiating minor compaction (all files) 2024-11-22T03:42:04,095 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HStore(1541): 4a7ce0cfb17f4dcc3dd24b2156963795/B is initiating minor compaction (all files) 2024-11-22T03:42:04,095 INFO [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4a7ce0cfb17f4dcc3dd24b2156963795/A in TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795. 2024-11-22T03:42:04,095 INFO [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4a7ce0cfb17f4dcc3dd24b2156963795/B in TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795. 2024-11-22T03:42:04,095 INFO [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/A/892770ca81db42de8adbb8f81e60a526, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/A/8e923d4205144abea3d6f5e110f7961b, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/A/44ab2d6996b046d493a4e6f63b3d09cd] into tmpdir=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/.tmp, totalSize=38.3 K 2024-11-22T03:42:04,095 INFO [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/B/031726fec90d4c01ae917ca441ec14d5, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/B/96ff445498f445dc8dcd512fb3da5c27, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/B/9bcaa526e82641cab5ce90d52f648996] into tmpdir=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/.tmp, totalSize=36.0 K 2024-11-22T03:42:04,096 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] compactions.Compactor(225): Compacting 892770ca81db42de8adbb8f81e60a526, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=188, earliestPutTs=1732246919817 2024-11-22T03:42:04,096 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] compactions.Compactor(225): Compacting 031726fec90d4c01ae917ca441ec14d5, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=188, earliestPutTs=1732246919817 2024-11-22T03:42:04,096 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8e923d4205144abea3d6f5e110f7961b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=196, earliestPutTs=1732246920639 2024-11-22T03:42:04,096 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] compactions.Compactor(225): Compacting 96ff445498f445dc8dcd512fb3da5c27, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=196, earliestPutTs=1732246920639 2024-11-22T03:42:04,096 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] compactions.Compactor(225): Compacting 44ab2d6996b046d493a4e6f63b3d09cd, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=228, earliestPutTs=1732246922061 2024-11-22T03:42:04,096 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] compactions.Compactor(225): Compacting 9bcaa526e82641cab5ce90d52f648996, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=228, earliestPutTs=1732246922061 2024-11-22T03:42:04,104 INFO [RS:0;c85114ed5096:42545-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4a7ce0cfb17f4dcc3dd24b2156963795#B#compaction#375 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T03:42:04,104 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/.tmp/B/07d6e42780ed433d9e8b80bb72c2d006 is 50, key is test_row_0/B:col10/1732246922061/Put/seqid=0 2024-11-22T03:42:04,107 INFO [RS:0;c85114ed5096:42545-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4a7ce0cfb17f4dcc3dd24b2156963795#A#compaction#376 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T03:42:04,107 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/.tmp/A/7731601f62e9420b8eb5bf7e2b49979e is 50, key is test_row_0/A:col10/1732246922061/Put/seqid=0 2024-11-22T03:42:04,110 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742265_1441 (size=12663) 2024-11-22T03:42:04,114 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/.tmp/B/07d6e42780ed433d9e8b80bb72c2d006 as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/B/07d6e42780ed433d9e8b80bb72c2d006 2024-11-22T03:42:04,118 INFO [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4a7ce0cfb17f4dcc3dd24b2156963795/B of 4a7ce0cfb17f4dcc3dd24b2156963795 into 07d6e42780ed433d9e8b80bb72c2d006(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T03:42:04,118 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4a7ce0cfb17f4dcc3dd24b2156963795: 2024-11-22T03:42:04,118 INFO [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., storeName=4a7ce0cfb17f4dcc3dd24b2156963795/B, priority=13, startTime=1732246924095; duration=0sec 2024-11-22T03:42:04,118 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T03:42:04,118 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4a7ce0cfb17f4dcc3dd24b2156963795:B 2024-11-22T03:42:04,119 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T03:42:04,119 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T03:42:04,119 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HStore(1541): 4a7ce0cfb17f4dcc3dd24b2156963795/C is initiating minor compaction (all files) 2024-11-22T03:42:04,119 INFO [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4a7ce0cfb17f4dcc3dd24b2156963795/C in TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795. 2024-11-22T03:42:04,119 INFO [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/C/bc04f226748a48668bb72884ac2ba244, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/C/a7152235b6094b9c8abcb0cf1c779870, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/C/3145934cf1ca4770afed9477b8de31de] into tmpdir=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/.tmp, totalSize=36.0 K 2024-11-22T03:42:04,120 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] compactions.Compactor(225): Compacting bc04f226748a48668bb72884ac2ba244, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=188, earliestPutTs=1732246919817 2024-11-22T03:42:04,120 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] compactions.Compactor(225): Compacting a7152235b6094b9c8abcb0cf1c779870, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=196, earliestPutTs=1732246920639 2024-11-22T03:42:04,120 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] compactions.Compactor(225): Compacting 3145934cf1ca4770afed9477b8de31de, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=228, earliestPutTs=1732246922061 2024-11-22T03:42:04,126 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742266_1442 (size=12663) 2024-11-22T03:42:04,127 INFO [RS:0;c85114ed5096:42545-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4a7ce0cfb17f4dcc3dd24b2156963795#C#compaction#377 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T03:42:04,127 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/.tmp/C/956367ad9f6d4534b9da56e248803cee is 50, key is test_row_0/C:col10/1732246922061/Put/seqid=0 2024-11-22T03:42:04,131 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742267_1443 (size=12663) 2024-11-22T03:42:04,132 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/.tmp/A/7731601f62e9420b8eb5bf7e2b49979e as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/A/7731601f62e9420b8eb5bf7e2b49979e 2024-11-22T03:42:04,139 INFO [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4a7ce0cfb17f4dcc3dd24b2156963795/A of 4a7ce0cfb17f4dcc3dd24b2156963795 into 7731601f62e9420b8eb5bf7e2b49979e(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T03:42:04,139 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4a7ce0cfb17f4dcc3dd24b2156963795: 2024-11-22T03:42:04,139 INFO [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., storeName=4a7ce0cfb17f4dcc3dd24b2156963795/A, priority=13, startTime=1732246924095; duration=0sec 2024-11-22T03:42:04,139 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T03:42:04,139 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4a7ce0cfb17f4dcc3dd24b2156963795:A 2024-11-22T03:42:04,141 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/.tmp/C/956367ad9f6d4534b9da56e248803cee as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/C/956367ad9f6d4534b9da56e248803cee 2024-11-22T03:42:04,151 INFO [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4a7ce0cfb17f4dcc3dd24b2156963795/C of 4a7ce0cfb17f4dcc3dd24b2156963795 into 956367ad9f6d4534b9da56e248803cee(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T03:42:04,151 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4a7ce0cfb17f4dcc3dd24b2156963795: 2024-11-22T03:42:04,151 INFO [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., storeName=4a7ce0cfb17f4dcc3dd24b2156963795/C, priority=13, startTime=1732246924095; duration=0sec 2024-11-22T03:42:04,151 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T03:42:04,151 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4a7ce0cfb17f4dcc3dd24b2156963795:C 2024-11-22T03:42:04,185 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42545 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-22T03:42:04,185 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795. 2024-11-22T03:42:04,185 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2902): Flushing 4a7ce0cfb17f4dcc3dd24b2156963795 3/3 column families, dataSize=26.84 KB heapSize=71.06 KB 2024-11-22T03:42:04,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/.tmp/A/6ec78fef2e624aa5af9b251465f1983f is 50, key is test_row_1/A:col10/1732246922842/Put/seqid=0 2024-11-22T03:42:04,197 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742268_1444 (size=9757) 2024-11-22T03:42:04,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(8855): Flush requested on 4a7ce0cfb17f4dcc3dd24b2156963795 2024-11-22T03:42:04,243 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795. as already flushing 2024-11-22T03:42:04,291 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:42:04,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40990 deadline: 1732246984288, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 2024-11-22T03:42:04,291 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:04,291 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:04,291 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:42:04,295 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:42:04,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40926 deadline: 1732246984290, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 2024-11-22T03:42:04,295 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:04,295 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:04,295 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:42:04,300 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:42:04,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40966 deadline: 1732246984291, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 2024-11-22T03:42:04,300 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:04,300 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:04,300 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:42:04,400 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:42:04,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40990 deadline: 1732246984398, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 2024-11-22T03:42:04,401 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:42:04,401 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:04,401 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:04,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40926 deadline: 1732246984398, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 2024-11-22T03:42:04,401 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:42:04,401 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:04,401 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:04,401 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:42:04,413 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:42:04,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40966 deadline: 1732246984408, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 2024-11-22T03:42:04,413 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:04,413 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:04,413 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:42:04,599 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=238 (bloomFilter=true), to=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/.tmp/A/6ec78fef2e624aa5af9b251465f1983f 2024-11-22T03:42:04,605 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/.tmp/B/c9723ccbbfe24071a0dc29c27912627d is 50, key is test_row_1/B:col10/1732246922842/Put/seqid=0 2024-11-22T03:42:04,611 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742269_1445 (size=9757) 2024-11-22T03:42:04,615 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:42:04,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40990 deadline: 1732246984609, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 2024-11-22T03:42:04,616 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:42:04,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40926 deadline: 1732246984609, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 2024-11-22T03:42:04,617 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:04,617 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:04,617 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:42:04,617 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:04,617 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:04,617 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:42:04,625 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:42:04,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40966 deadline: 1732246984619, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 2024-11-22T03:42:04,626 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:04,626 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:04,626 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:42:04,933 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:42:04,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40926 deadline: 1732246984928, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 2024-11-22T03:42:04,934 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:42:04,934 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:04,934 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:04,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40990 deadline: 1732246984928, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 2024-11-22T03:42:04,934 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:42:04,934 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:04,934 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:04,934 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:42:04,942 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:42:04,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40966 deadline: 1732246984939, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 2024-11-22T03:42:04,943 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:04,943 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:04,943 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:42:04,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-22T03:42:05,012 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=238 (bloomFilter=true), to=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/.tmp/B/c9723ccbbfe24071a0dc29c27912627d 2024-11-22T03:42:05,014 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:42:05,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40962 deadline: 1732246985008, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 2024-11-22T03:42:05,014 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to c85114ed5096,42545,1732246786482 for 'test_row_0' in 4a7ce0cfb17f4dcc3dd24b2156963795 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2165 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-22T03:42:05,015 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:05,015 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:05,015 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:42:05,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/.tmp/C/6e42e52f3dcb4a38b274a0e52fe6c7eb is 50, key is test_row_1/C:col10/1732246922842/Put/seqid=0 2024-11-22T03:42:05,019 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742270_1446 (size=9757) 2024-11-22T03:42:05,023 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:42:05,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41022 deadline: 1732246985019, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 2024-11-22T03:42:05,024 WARN [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRpcRetryingCaller(177): Call to c85114ed5096,42545,1732246786482 for 'test_row_0' in 4a7ce0cfb17f4dcc3dd24b2156963795 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2172 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-22T03:42:05,024 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:05,024 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:05,024 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:42:05,420 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=238 (bloomFilter=true), to=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/.tmp/C/6e42e52f3dcb4a38b274a0e52fe6c7eb 2024-11-22T03:42:05,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/.tmp/A/6ec78fef2e624aa5af9b251465f1983f as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/A/6ec78fef2e624aa5af9b251465f1983f 2024-11-22T03:42:05,426 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/A/6ec78fef2e624aa5af9b251465f1983f, entries=100, sequenceid=238, filesize=9.5 K 2024-11-22T03:42:05,426 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/.tmp/B/c9723ccbbfe24071a0dc29c27912627d as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/B/c9723ccbbfe24071a0dc29c27912627d 2024-11-22T03:42:05,429 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/B/c9723ccbbfe24071a0dc29c27912627d, entries=100, sequenceid=238, filesize=9.5 K 2024-11-22T03:42:05,430 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/.tmp/C/6e42e52f3dcb4a38b274a0e52fe6c7eb as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/C/6e42e52f3dcb4a38b274a0e52fe6c7eb 2024-11-22T03:42:05,432 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/C/6e42e52f3dcb4a38b274a0e52fe6c7eb, entries=100, sequenceid=238, filesize=9.5 K 2024-11-22T03:42:05,433 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~71.02 KB/72720, currentSize=174.43 KB/178620 for 4a7ce0cfb17f4dcc3dd24b2156963795 in 1248ms, sequenceid=238, compaction requested=false 2024-11-22T03:42:05,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2603): Flush status journal for 4a7ce0cfb17f4dcc3dd24b2156963795: 2024-11-22T03:42:05,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795. 2024-11-22T03:42:05,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=132 2024-11-22T03:42:05,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42627 {}] master.HMaster(4169): Remote procedure done, pid=132 2024-11-22T03:42:05,435 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=132, resume processing ppid=131 2024-11-22T03:42:05,435 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=132, ppid=131, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.6220 sec 2024-11-22T03:42:05,436 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=131, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees in 2.6260 sec 2024-11-22T03:42:05,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(8855): Flush requested on 4a7ce0cfb17f4dcc3dd24b2156963795 2024-11-22T03:42:05,445 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-22T03:42:05,445 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4a7ce0cfb17f4dcc3dd24b2156963795 3/3 column families, dataSize=187.85 KB heapSize=492.94 KB 2024-11-22T03:42:05,449 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/.tmp/A/a612e4acab9e45cca2fb4976211a51bb is 50, key is test_row_0/A:col10/1732246924289/Put/seqid=0 2024-11-22T03:42:05,457 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742271_1447 (size=17181) 2024-11-22T03:42:05,457 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:42:05,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40926 deadline: 1732246985454, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 2024-11-22T03:42:05,457 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:05,457 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:05,457 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:42:05,462 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:42:05,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40990 deadline: 1732246985455, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 2024-11-22T03:42:05,462 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:42:05,462 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:05,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40966 deadline: 1732246985455, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 2024-11-22T03:42:05,463 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:05,463 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:42:05,463 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:05,463 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:05,463 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:42:05,571 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:42:05,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40926 deadline: 1732246985568, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 2024-11-22T03:42:05,571 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:42:05,572 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:05,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40990 deadline: 1732246985568, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 2024-11-22T03:42:05,572 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:05,572 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:42:05,572 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:05,572 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:42:05,572 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:05,572 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:42:05,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40966 deadline: 1732246985569, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 2024-11-22T03:42:05,572 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:05,572 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:05,572 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:42:05,781 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:42:05,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40926 deadline: 1732246985778, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 2024-11-22T03:42:05,782 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:05,782 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:42:05,782 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:05,782 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:42:05,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40990 deadline: 1732246985778, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 2024-11-22T03:42:05,782 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:05,782 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:05,782 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:42:05,782 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:42:05,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40966 deadline: 1732246985778, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 2024-11-22T03:42:05,782 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:05,782 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:05,783 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:42:05,857 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=62.62 KB at sequenceid=269 (bloomFilter=true), to=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/.tmp/A/a612e4acab9e45cca2fb4976211a51bb 2024-11-22T03:42:05,863 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/.tmp/B/985850e0381e47ad878559d3c83cf4aa is 50, key is test_row_0/B:col10/1732246924289/Put/seqid=0 2024-11-22T03:42:05,867 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742272_1448 (size=12301) 2024-11-22T03:42:06,092 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:42:06,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40926 deadline: 1732246986089, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 2024-11-22T03:42:06,092 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:06,092 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:42:06,093 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:06,093 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:42:06,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40990 deadline: 1732246986089, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 2024-11-22T03:42:06,093 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:06,093 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:06,093 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:42:06,094 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:42:06,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40966 deadline: 1732246986089, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 2024-11-22T03:42:06,095 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:06,095 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:06,095 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:42:06,267 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=62.62 KB at sequenceid=269 (bloomFilter=true), to=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/.tmp/B/985850e0381e47ad878559d3c83cf4aa 2024-11-22T03:42:06,274 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/.tmp/C/603ff7b096b547538d25c14e5ea6f147 is 50, key is test_row_0/C:col10/1732246924289/Put/seqid=0 2024-11-22T03:42:06,281 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742273_1449 (size=12301) 2024-11-22T03:42:06,281 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=62.62 KB at sequenceid=269 (bloomFilter=true), to=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/.tmp/C/603ff7b096b547538d25c14e5ea6f147 2024-11-22T03:42:06,285 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/.tmp/A/a612e4acab9e45cca2fb4976211a51bb as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/A/a612e4acab9e45cca2fb4976211a51bb 2024-11-22T03:42:06,288 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/A/a612e4acab9e45cca2fb4976211a51bb, entries=250, sequenceid=269, filesize=16.8 K 2024-11-22T03:42:06,289 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/.tmp/B/985850e0381e47ad878559d3c83cf4aa as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/B/985850e0381e47ad878559d3c83cf4aa 2024-11-22T03:42:06,292 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/B/985850e0381e47ad878559d3c83cf4aa, entries=150, sequenceid=269, filesize=12.0 K 2024-11-22T03:42:06,292 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/.tmp/C/603ff7b096b547538d25c14e5ea6f147 as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/C/603ff7b096b547538d25c14e5ea6f147 2024-11-22T03:42:06,296 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/C/603ff7b096b547538d25c14e5ea6f147, entries=150, sequenceid=269, filesize=12.0 K 2024-11-22T03:42:06,297 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~187.85 KB/192360, heapSize ~492.89 KB/504720, currentSize=20.13 KB/20610 for 4a7ce0cfb17f4dcc3dd24b2156963795 in 852ms, sequenceid=269, compaction requested=true 2024-11-22T03:42:06,297 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4a7ce0cfb17f4dcc3dd24b2156963795: 2024-11-22T03:42:06,298 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T03:42:06,298 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4a7ce0cfb17f4dcc3dd24b2156963795:A, priority=-2147483648, current under compaction store size is 1 2024-11-22T03:42:06,298 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T03:42:06,298 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4a7ce0cfb17f4dcc3dd24b2156963795:B, priority=-2147483648, current under compaction store size is 2 2024-11-22T03:42:06,298 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T03:42:06,298 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T03:42:06,298 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4a7ce0cfb17f4dcc3dd24b2156963795:C, priority=-2147483648, current under compaction store size is 3 2024-11-22T03:42:06,298 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T03:42:06,299 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39601 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T03:42:06,299 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.HStore(1541): 4a7ce0cfb17f4dcc3dd24b2156963795/A is initiating minor compaction (all files) 2024-11-22T03:42:06,299 INFO [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4a7ce0cfb17f4dcc3dd24b2156963795/A in TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795. 2024-11-22T03:42:06,299 INFO [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/A/7731601f62e9420b8eb5bf7e2b49979e, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/A/6ec78fef2e624aa5af9b251465f1983f, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/A/a612e4acab9e45cca2fb4976211a51bb] into tmpdir=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/.tmp, totalSize=38.7 K 2024-11-22T03:42:06,300 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34721 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T03:42:06,300 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HStore(1541): 4a7ce0cfb17f4dcc3dd24b2156963795/B is initiating minor compaction (all files) 2024-11-22T03:42:06,300 INFO [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4a7ce0cfb17f4dcc3dd24b2156963795/B in TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795. 2024-11-22T03:42:06,300 INFO [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/B/07d6e42780ed433d9e8b80bb72c2d006, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/B/c9723ccbbfe24071a0dc29c27912627d, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/B/985850e0381e47ad878559d3c83cf4aa] into tmpdir=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/.tmp, totalSize=33.9 K 2024-11-22T03:42:06,300 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7731601f62e9420b8eb5bf7e2b49979e, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=228, earliestPutTs=1732246922061 2024-11-22T03:42:06,300 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] compactions.Compactor(225): Compacting 07d6e42780ed433d9e8b80bb72c2d006, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=228, earliestPutTs=1732246922061 2024-11-22T03:42:06,300 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6ec78fef2e624aa5af9b251465f1983f, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=238, earliestPutTs=1732246922842 2024-11-22T03:42:06,301 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] compactions.Compactor(225): Compacting c9723ccbbfe24071a0dc29c27912627d, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=238, earliestPutTs=1732246922842 2024-11-22T03:42:06,301 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] compactions.Compactor(225): Compacting a612e4acab9e45cca2fb4976211a51bb, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=269, earliestPutTs=1732246924279 2024-11-22T03:42:06,301 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] compactions.Compactor(225): Compacting 985850e0381e47ad878559d3c83cf4aa, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=269, earliestPutTs=1732246924288 2024-11-22T03:42:06,310 INFO [RS:0;c85114ed5096:42545-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4a7ce0cfb17f4dcc3dd24b2156963795#B#compaction#384 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-22T03:42:06,310 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/.tmp/B/9b17341d9a7c4c808ca45da794e72e12 is 50, key is test_row_0/B:col10/1732246924289/Put/seqid=0 2024-11-22T03:42:06,311 INFO [RS:0;c85114ed5096:42545-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4a7ce0cfb17f4dcc3dd24b2156963795#A#compaction#385 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T03:42:06,311 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/.tmp/A/b0ff8e7b62d44bcebcec077f4aeeea18 is 50, key is test_row_0/A:col10/1732246924289/Put/seqid=0 2024-11-22T03:42:06,344 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742274_1450 (size=12915) 2024-11-22T03:42:06,352 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/.tmp/B/9b17341d9a7c4c808ca45da794e72e12 as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/B/9b17341d9a7c4c808ca45da794e72e12 2024-11-22T03:42:06,359 INFO [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4a7ce0cfb17f4dcc3dd24b2156963795/B of 4a7ce0cfb17f4dcc3dd24b2156963795 into 9b17341d9a7c4c808ca45da794e72e12(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T03:42:06,359 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4a7ce0cfb17f4dcc3dd24b2156963795: 2024-11-22T03:42:06,359 INFO [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., storeName=4a7ce0cfb17f4dcc3dd24b2156963795/B, priority=13, startTime=1732246926298; duration=0sec 2024-11-22T03:42:06,361 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T03:42:06,361 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4a7ce0cfb17f4dcc3dd24b2156963795:B 2024-11-22T03:42:06,361 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T03:42:06,362 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34721 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T03:42:06,362 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HStore(1541): 4a7ce0cfb17f4dcc3dd24b2156963795/C is initiating minor compaction (all files) 2024-11-22T03:42:06,362 INFO [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4a7ce0cfb17f4dcc3dd24b2156963795/C in TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795. 2024-11-22T03:42:06,362 INFO [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/C/956367ad9f6d4534b9da56e248803cee, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/C/6e42e52f3dcb4a38b274a0e52fe6c7eb, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/C/603ff7b096b547538d25c14e5ea6f147] into tmpdir=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/.tmp, totalSize=33.9 K 2024-11-22T03:42:06,362 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] compactions.Compactor(225): Compacting 956367ad9f6d4534b9da56e248803cee, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=228, earliestPutTs=1732246922061 2024-11-22T03:42:06,362 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] compactions.Compactor(225): Compacting 6e42e52f3dcb4a38b274a0e52fe6c7eb, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=238, earliestPutTs=1732246922842 2024-11-22T03:42:06,363 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] compactions.Compactor(225): Compacting 603ff7b096b547538d25c14e5ea6f147, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=269, earliestPutTs=1732246924288 2024-11-22T03:42:06,380 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742275_1451 (size=12915) 2024-11-22T03:42:06,396 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/.tmp/A/b0ff8e7b62d44bcebcec077f4aeeea18 as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/A/b0ff8e7b62d44bcebcec077f4aeeea18 2024-11-22T03:42:06,401 INFO [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4a7ce0cfb17f4dcc3dd24b2156963795/A of 4a7ce0cfb17f4dcc3dd24b2156963795 into b0ff8e7b62d44bcebcec077f4aeeea18(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T03:42:06,401 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4a7ce0cfb17f4dcc3dd24b2156963795: 2024-11-22T03:42:06,401 INFO [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., storeName=4a7ce0cfb17f4dcc3dd24b2156963795/A, priority=13, startTime=1732246926297; duration=0sec 2024-11-22T03:42:06,401 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T03:42:06,401 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4a7ce0cfb17f4dcc3dd24b2156963795:A 2024-11-22T03:42:06,412 INFO [RS:0;c85114ed5096:42545-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4a7ce0cfb17f4dcc3dd24b2156963795#C#compaction#386 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T03:42:06,413 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/.tmp/C/39c603a41ce04cfab5ed3a164df485ab is 50, key is test_row_0/C:col10/1732246924289/Put/seqid=0 2024-11-22T03:42:06,437 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742276_1452 (size=12915) 2024-11-22T03:42:06,444 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/.tmp/C/39c603a41ce04cfab5ed3a164df485ab as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/C/39c603a41ce04cfab5ed3a164df485ab 2024-11-22T03:42:06,451 INFO [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4a7ce0cfb17f4dcc3dd24b2156963795/C of 4a7ce0cfb17f4dcc3dd24b2156963795 into 39c603a41ce04cfab5ed3a164df485ab(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T03:42:06,451 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4a7ce0cfb17f4dcc3dd24b2156963795: 2024-11-22T03:42:06,451 INFO [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., storeName=4a7ce0cfb17f4dcc3dd24b2156963795/C, priority=13, startTime=1732246926298; duration=0sec 2024-11-22T03:42:06,451 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T03:42:06,451 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4a7ce0cfb17f4dcc3dd24b2156963795:C 2024-11-22T03:42:06,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(8855): Flush requested on 4a7ce0cfb17f4dcc3dd24b2156963795 2024-11-22T03:42:06,605 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-22T03:42:06,605 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4a7ce0cfb17f4dcc3dd24b2156963795 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-22T03:42:06,608 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/.tmp/A/2e804fefc6474369a9c4f8fa9d19d814 is 50, key is test_row_0/A:col10/1732246926604/Put/seqid=0 2024-11-22T03:42:06,612 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742277_1453 (size=17181) 2024-11-22T03:42:06,658 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:42:06,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40926 deadline: 1732246986652, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 2024-11-22T03:42:06,658 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:06,658 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:06,658 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:42:06,662 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:42:06,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40966 deadline: 1732246986653, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 2024-11-22T03:42:06,664 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:06,664 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:06,664 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:42:06,666 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:42:06,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40990 deadline: 1732246986656, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 2024-11-22T03:42:06,667 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:06,667 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:06,667 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:42:06,770 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:42:06,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40926 deadline: 1732246986768, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 2024-11-22T03:42:06,771 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:42:06,771 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:06,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40966 deadline: 1732246986768, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 2024-11-22T03:42:06,771 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:06,771 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:42:06,771 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:06,771 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:42:06,771 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:06,771 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:42:06,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40990 deadline: 1732246986768, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 2024-11-22T03:42:06,771 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:06,771 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:06,771 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:42:06,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-22T03:42:06,969 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-22T03:42:06,969 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42627 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-22T03:42:06,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42627 {}] procedure2.ProcedureExecutor(1139): Stored pid=133, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees 2024-11-22T03:42:06,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-22T03:42:06,971 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=133, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-22T03:42:06,972 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=133, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-22T03:42:06,972 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=134, ppid=133, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-22T03:42:06,980 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:42:06,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40926 deadline: 1732246986978, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 2024-11-22T03:42:06,981 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:06,981 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:42:06,981 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:06,981 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:42:06,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40966 deadline: 1732246986979, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 2024-11-22T03:42:06,981 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:06,981 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:06,981 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:42:06,981 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:42:06,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40990 deadline: 1732246986979, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 2024-11-22T03:42:06,981 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:06,981 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:06,981 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:42:07,013 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=283 (bloomFilter=true), to=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/.tmp/A/2e804fefc6474369a9c4f8fa9d19d814 2024-11-22T03:42:07,022 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/.tmp/B/9f40ad9c40204d738e211c2f0eacf91e is 50, key is test_row_0/B:col10/1732246926604/Put/seqid=0 2024-11-22T03:42:07,034 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:42:07,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40962 deadline: 1732246987028, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 2024-11-22T03:42:07,034 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:42:07,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41022 deadline: 1732246987028, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 2024-11-22T03:42:07,034 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to c85114ed5096,42545,1732246786482 for 'test_row_0' in 4a7ce0cfb17f4dcc3dd24b2156963795 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4185 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-22T03:42:07,035 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:07,035 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:07,035 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:42:07,035 WARN [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRpcRetryingCaller(177): Call to c85114ed5096,42545,1732246786482 for 'test_row_0' in 4a7ce0cfb17f4dcc3dd24b2156963795 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4184 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-22T03:42:07,035 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:07,035 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:07,035 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:42:07,036 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742278_1454 (size=12301) 2024-11-22T03:42:07,037 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=283 (bloomFilter=true), to=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/.tmp/B/9f40ad9c40204d738e211c2f0eacf91e 2024-11-22T03:42:07,045 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/.tmp/C/311ebb17261249979ffa70985e116e07 is 50, key is test_row_0/C:col10/1732246926604/Put/seqid=0 2024-11-22T03:42:07,051 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742279_1455 (size=12301) 2024-11-22T03:42:07,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-22T03:42:07,125 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42545 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-22T03:42:07,125 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795. 2024-11-22T03:42:07,125 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795. as already flushing 2024-11-22T03:42:07,125 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795. 2024-11-22T03:42:07,125 ERROR [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => 4a7ce0cfb17f4dcc3dd24b2156963795, NAME => 'TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:42:07,126 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => 4a7ce0cfb17f4dcc3dd24b2156963795, NAME => 'TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:42:07,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42627 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4a7ce0cfb17f4dcc3dd24b2156963795, NAME => 'TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4a7ce0cfb17f4dcc3dd24b2156963795, NAME => 'TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:42:07,278 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42545 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-22T03:42:07,278 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795. 2024-11-22T03:42:07,278 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795. as already flushing 2024-11-22T03:42:07,278 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795. 2024-11-22T03:42:07,278 ERROR [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => 4a7ce0cfb17f4dcc3dd24b2156963795, NAME => 'TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:42:07,278 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => 4a7ce0cfb17f4dcc3dd24b2156963795, NAME => 'TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:42:07,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42627 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4a7ce0cfb17f4dcc3dd24b2156963795, NAME => 'TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4a7ce0cfb17f4dcc3dd24b2156963795, NAME => 'TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:42:07,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-22T03:42:07,297 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:42:07,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40926 deadline: 1732246987289, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 2024-11-22T03:42:07,298 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:42:07,298 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:07,298 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:07,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40966 deadline: 1732246987289, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 2024-11-22T03:42:07,298 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:42:07,298 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:07,298 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:42:07,298 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:07,298 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:42:07,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40990 deadline: 1732246987289, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 2024-11-22T03:42:07,298 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:07,298 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:07,298 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:42:07,430 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42545 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-22T03:42:07,430 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795. 2024-11-22T03:42:07,430 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795. as already flushing 2024-11-22T03:42:07,430 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795. 2024-11-22T03:42:07,430 ERROR [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => 4a7ce0cfb17f4dcc3dd24b2156963795, NAME => 'TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:42:07,430 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => 4a7ce0cfb17f4dcc3dd24b2156963795, NAME => 'TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:42:07,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42627 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4a7ce0cfb17f4dcc3dd24b2156963795, NAME => 'TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4a7ce0cfb17f4dcc3dd24b2156963795, NAME => 'TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:42:07,452 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=283 (bloomFilter=true), to=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/.tmp/C/311ebb17261249979ffa70985e116e07 2024-11-22T03:42:07,455 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/.tmp/A/2e804fefc6474369a9c4f8fa9d19d814 as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/A/2e804fefc6474369a9c4f8fa9d19d814 2024-11-22T03:42:07,457 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/A/2e804fefc6474369a9c4f8fa9d19d814, entries=250, sequenceid=283, filesize=16.8 K 2024-11-22T03:42:07,458 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/.tmp/B/9f40ad9c40204d738e211c2f0eacf91e as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/B/9f40ad9c40204d738e211c2f0eacf91e 2024-11-22T03:42:07,460 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/B/9f40ad9c40204d738e211c2f0eacf91e, entries=150, sequenceid=283, filesize=12.0 K 2024-11-22T03:42:07,461 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/.tmp/C/311ebb17261249979ffa70985e116e07 as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/C/311ebb17261249979ffa70985e116e07 2024-11-22T03:42:07,463 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/C/311ebb17261249979ffa70985e116e07, entries=150, sequenceid=283, filesize=12.0 K 2024-11-22T03:42:07,464 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 4a7ce0cfb17f4dcc3dd24b2156963795 in 859ms, sequenceid=283, compaction requested=false 2024-11-22T03:42:07,464 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4a7ce0cfb17f4dcc3dd24b2156963795: 2024-11-22T03:42:07,582 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42545 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-22T03:42:07,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795. 2024-11-22T03:42:07,583 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2902): Flushing 4a7ce0cfb17f4dcc3dd24b2156963795 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-22T03:42:07,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/.tmp/A/6cad706e30104a84b7a229e70c4c778d is 50, key is test_row_0/A:col10/1732246926633/Put/seqid=0 2024-11-22T03:42:07,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-22T03:42:07,600 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742280_1456 (size=12301) 2024-11-22T03:42:07,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(8855): Flush requested on 4a7ce0cfb17f4dcc3dd24b2156963795 2024-11-22T03:42:07,811 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795. as already flushing 2024-11-22T03:42:07,831 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:42:07,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40990 deadline: 1732246987820, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 2024-11-22T03:42:07,831 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:42:07,831 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:07,831 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:07,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40926 deadline: 1732246987822, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 2024-11-22T03:42:07,831 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:42:07,831 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:07,831 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:07,831 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:42:07,832 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:42:07,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40966 deadline: 1732246987829, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 2024-11-22T03:42:07,832 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:07,832 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:07,832 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:42:07,944 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:42:07,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40990 deadline: 1732246987939, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 2024-11-22T03:42:07,945 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:42:07,945 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:07,945 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:07,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40926 deadline: 1732246987939, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 2024-11-22T03:42:07,945 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:42:07,945 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:07,945 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:07,945 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:42:07,945 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:42:07,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40966 deadline: 1732246987939, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 2024-11-22T03:42:07,945 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:07,945 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:07,945 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:42:08,000 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=308 (bloomFilter=true), to=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/.tmp/A/6cad706e30104a84b7a229e70c4c778d 2024-11-22T03:42:08,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/.tmp/B/383df294c68848c6a474c4249586c1a2 is 50, key is test_row_0/B:col10/1732246926633/Put/seqid=0 2024-11-22T03:42:08,010 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742281_1457 (size=12301) 2024-11-22T03:42:08,013 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=308 (bloomFilter=true), to=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/.tmp/B/383df294c68848c6a474c4249586c1a2 2024-11-22T03:42:08,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/.tmp/C/1a0a6472d57b4d0da4bea93e692ca07c is 50, key is test_row_0/C:col10/1732246926633/Put/seqid=0 2024-11-22T03:42:08,028 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742282_1458 (size=12301) 2024-11-22T03:42:08,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-22T03:42:08,155 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:42:08,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40990 deadline: 1732246988149, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 2024-11-22T03:42:08,155 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:42:08,155 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:08,155 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:08,155 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:42:08,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40966 deadline: 1732246988149, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 2024-11-22T03:42:08,156 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:08,156 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:08,156 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:42:08,156 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:42:08,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40926 deadline: 1732246988149, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 2024-11-22T03:42:08,157 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:08,157 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:08,157 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:42:08,429 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=308 (bloomFilter=true), to=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/.tmp/C/1a0a6472d57b4d0da4bea93e692ca07c 2024-11-22T03:42:08,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/.tmp/A/6cad706e30104a84b7a229e70c4c778d as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/A/6cad706e30104a84b7a229e70c4c778d 2024-11-22T03:42:08,436 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/A/6cad706e30104a84b7a229e70c4c778d, entries=150, sequenceid=308, filesize=12.0 K 2024-11-22T03:42:08,437 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/.tmp/B/383df294c68848c6a474c4249586c1a2 as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/B/383df294c68848c6a474c4249586c1a2 2024-11-22T03:42:08,440 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/B/383df294c68848c6a474c4249586c1a2, entries=150, sequenceid=308, filesize=12.0 K 2024-11-22T03:42:08,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/.tmp/C/1a0a6472d57b4d0da4bea93e692ca07c as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/C/1a0a6472d57b4d0da4bea93e692ca07c 2024-11-22T03:42:08,443 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/C/1a0a6472d57b4d0da4bea93e692ca07c, entries=150, sequenceid=308, filesize=12.0 K 2024-11-22T03:42:08,443 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 4a7ce0cfb17f4dcc3dd24b2156963795 in 860ms, sequenceid=308, compaction requested=true 2024-11-22T03:42:08,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2603): Flush status journal for 4a7ce0cfb17f4dcc3dd24b2156963795: 2024-11-22T03:42:08,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795. 2024-11-22T03:42:08,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=134 2024-11-22T03:42:08,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42627 {}] master.HMaster(4169): Remote procedure done, pid=134 2024-11-22T03:42:08,446 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=134, resume processing ppid=133 2024-11-22T03:42:08,446 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=134, ppid=133, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4720 sec 2024-11-22T03:42:08,448 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=133, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees in 1.4780 sec 2024-11-22T03:42:08,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(8855): Flush requested on 4a7ce0cfb17f4dcc3dd24b2156963795 2024-11-22T03:42:08,463 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-22T03:42:08,463 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4a7ce0cfb17f4dcc3dd24b2156963795 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-22T03:42:08,466 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/.tmp/A/ef2a06fcf77541979a97da3f08167b8c is 50, key is test_row_0/A:col10/1732246927817/Put/seqid=0 2024-11-22T03:42:08,469 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742283_1459 (size=14741) 2024-11-22T03:42:08,469 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=320 (bloomFilter=true), to=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/.tmp/A/ef2a06fcf77541979a97da3f08167b8c 2024-11-22T03:42:08,476 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/.tmp/B/2cecc979d6e748f497e69eb98b8e5e90 is 50, key is test_row_0/B:col10/1732246927817/Put/seqid=0 2024-11-22T03:42:08,482 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742284_1460 (size=12301) 2024-11-22T03:42:08,528 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:42:08,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40926 deadline: 1732246988516, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 2024-11-22T03:42:08,529 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:08,529 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:42:08,529 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:08,529 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:42:08,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40990 deadline: 1732246988520, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 2024-11-22T03:42:08,529 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:08,529 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:42:08,529 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:08,529 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:42:08,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40966 deadline: 1732246988527, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 2024-11-22T03:42:08,529 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:08,529 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:08,529 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:42:08,645 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:42:08,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40926 deadline: 1732246988638, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 2024-11-22T03:42:08,646 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:42:08,646 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:42:08,646 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:08,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40966 deadline: 1732246988638, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 2024-11-22T03:42:08,646 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:08,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40990 deadline: 1732246988638, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 2024-11-22T03:42:08,646 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:42:08,646 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:08,646 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:08,646 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:08,646 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:08,646 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:42:08,646 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:42:08,851 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:42:08,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40966 deadline: 1732246988848, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 2024-11-22T03:42:08,852 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:08,852 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:08,852 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:42:08,865 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:42:08,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40926 deadline: 1732246988858, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 2024-11-22T03:42:08,865 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:42:08,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40990 deadline: 1732246988858, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 2024-11-22T03:42:08,865 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:08,865 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:08,865 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:42:08,866 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:08,866 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:08,866 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:42:08,883 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=320 (bloomFilter=true), to=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/.tmp/B/2cecc979d6e748f497e69eb98b8e5e90 2024-11-22T03:42:08,888 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/.tmp/C/392b18f8879b47e3a54467440a77e8bb is 50, key is test_row_0/C:col10/1732246927817/Put/seqid=0 2024-11-22T03:42:08,890 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742285_1461 (size=12301) 2024-11-22T03:42:09,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-22T03:42:09,129 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-22T03:42:09,129 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42627 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-22T03:42:09,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42627 {}] procedure2.ProcedureExecutor(1139): Stored pid=135, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees 2024-11-22T03:42:09,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-22T03:42:09,130 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=135, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-22T03:42:09,131 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=135, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-22T03:42:09,131 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=136, ppid=135, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-22T03:42:09,162 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:42:09,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40966 deadline: 1732246989159, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 2024-11-22T03:42:09,162 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:09,162 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:09,162 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:42:09,176 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:42:09,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40926 deadline: 1732246989169, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 2024-11-22T03:42:09,177 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:09,177 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:09,177 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:42:09,185 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:42:09,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40990 deadline: 1732246989178, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 2024-11-22T03:42:09,186 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:09,186 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:09,186 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:42:09,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-22T03:42:09,283 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42545 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-22T03:42:09,283 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795. 2024-11-22T03:42:09,283 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795. as already flushing 2024-11-22T03:42:09,283 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795. 2024-11-22T03:42:09,283 ERROR [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => 4a7ce0cfb17f4dcc3dd24b2156963795, NAME => 'TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:42:09,283 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => 4a7ce0cfb17f4dcc3dd24b2156963795, NAME => 'TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:42:09,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42627 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4a7ce0cfb17f4dcc3dd24b2156963795, NAME => 'TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4a7ce0cfb17f4dcc3dd24b2156963795, NAME => 'TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:42:09,291 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=320 (bloomFilter=true), to=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/.tmp/C/392b18f8879b47e3a54467440a77e8bb 2024-11-22T03:42:09,294 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/.tmp/A/ef2a06fcf77541979a97da3f08167b8c as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/A/ef2a06fcf77541979a97da3f08167b8c 2024-11-22T03:42:09,296 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/A/ef2a06fcf77541979a97da3f08167b8c, entries=200, sequenceid=320, filesize=14.4 K 2024-11-22T03:42:09,297 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/.tmp/B/2cecc979d6e748f497e69eb98b8e5e90 as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/B/2cecc979d6e748f497e69eb98b8e5e90 2024-11-22T03:42:09,300 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/B/2cecc979d6e748f497e69eb98b8e5e90, entries=150, sequenceid=320, filesize=12.0 K 2024-11-22T03:42:09,300 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/.tmp/C/392b18f8879b47e3a54467440a77e8bb as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/C/392b18f8879b47e3a54467440a77e8bb 2024-11-22T03:42:09,303 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/C/392b18f8879b47e3a54467440a77e8bb, entries=150, sequenceid=320, filesize=12.0 K 2024-11-22T03:42:09,303 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 4a7ce0cfb17f4dcc3dd24b2156963795 in 840ms, sequenceid=320, compaction requested=true 2024-11-22T03:42:09,303 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4a7ce0cfb17f4dcc3dd24b2156963795: 2024-11-22T03:42:09,303 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4a7ce0cfb17f4dcc3dd24b2156963795:A, priority=-2147483648, current under compaction store size is 1 2024-11-22T03:42:09,303 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T03:42:09,303 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-22T03:42:09,303 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4a7ce0cfb17f4dcc3dd24b2156963795:B, priority=-2147483648, current under compaction store size is 2 2024-11-22T03:42:09,303 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T03:42:09,303 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-22T03:42:09,303 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4a7ce0cfb17f4dcc3dd24b2156963795:C, priority=-2147483648, current under compaction store size is 3 2024-11-22T03:42:09,303 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T03:42:09,304 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 57138 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-22T03:42:09,304 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.HStore(1541): 4a7ce0cfb17f4dcc3dd24b2156963795/A is initiating minor compaction (all files) 2024-11-22T03:42:09,304 INFO [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4a7ce0cfb17f4dcc3dd24b2156963795/A in TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795. 2024-11-22T03:42:09,304 INFO [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/A/b0ff8e7b62d44bcebcec077f4aeeea18, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/A/2e804fefc6474369a9c4f8fa9d19d814, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/A/6cad706e30104a84b7a229e70c4c778d, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/A/ef2a06fcf77541979a97da3f08167b8c] into tmpdir=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/.tmp, totalSize=55.8 K 2024-11-22T03:42:09,305 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49818 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-22T03:42:09,305 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HStore(1541): 4a7ce0cfb17f4dcc3dd24b2156963795/B is initiating minor compaction (all files) 2024-11-22T03:42:09,305 INFO [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4a7ce0cfb17f4dcc3dd24b2156963795/B in TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795. 2024-11-22T03:42:09,305 INFO [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/B/9b17341d9a7c4c808ca45da794e72e12, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/B/9f40ad9c40204d738e211c2f0eacf91e, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/B/383df294c68848c6a474c4249586c1a2, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/B/2cecc979d6e748f497e69eb98b8e5e90] into tmpdir=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/.tmp, totalSize=48.7 K 2024-11-22T03:42:09,306 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] compactions.Compactor(225): Compacting 9b17341d9a7c4c808ca45da794e72e12, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=269, earliestPutTs=1732246924288 2024-11-22T03:42:09,306 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] compactions.Compactor(225): Compacting b0ff8e7b62d44bcebcec077f4aeeea18, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=269, earliestPutTs=1732246924288 2024-11-22T03:42:09,306 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2e804fefc6474369a9c4f8fa9d19d814, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=283, earliestPutTs=1732246926599 2024-11-22T03:42:09,306 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] compactions.Compactor(225): Compacting 9f40ad9c40204d738e211c2f0eacf91e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=283, earliestPutTs=1732246926599 2024-11-22T03:42:09,307 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6cad706e30104a84b7a229e70c4c778d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=308, earliestPutTs=1732246926633 2024-11-22T03:42:09,307 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] compactions.Compactor(225): Compacting 383df294c68848c6a474c4249586c1a2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=308, earliestPutTs=1732246926633 2024-11-22T03:42:09,307 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] compactions.Compactor(225): Compacting ef2a06fcf77541979a97da3f08167b8c, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=320, earliestPutTs=1732246927811 2024-11-22T03:42:09,307 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] compactions.Compactor(225): Compacting 2cecc979d6e748f497e69eb98b8e5e90, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=320, earliestPutTs=1732246927811 2024-11-22T03:42:09,313 INFO [RS:0;c85114ed5096:42545-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4a7ce0cfb17f4dcc3dd24b2156963795#B#compaction#396 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T03:42:09,314 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/.tmp/B/34c1617ac80442279a56cf8de53a92a8 is 50, key is test_row_0/B:col10/1732246927817/Put/seqid=0 2024-11-22T03:42:09,315 INFO [RS:0;c85114ed5096:42545-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4a7ce0cfb17f4dcc3dd24b2156963795#A#compaction#397 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T03:42:09,316 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/.tmp/A/bd2e8314cd6a432eaf51901bec5f9a4e is 50, key is test_row_0/A:col10/1732246927817/Put/seqid=0 2024-11-22T03:42:09,322 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742286_1462 (size=13051) 2024-11-22T03:42:09,331 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742287_1463 (size=13051) 2024-11-22T03:42:09,435 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42545 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-22T03:42:09,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795. 2024-11-22T03:42:09,435 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2902): Flushing 4a7ce0cfb17f4dcc3dd24b2156963795 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-22T03:42:09,439 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/.tmp/A/442b6a0715154852b2ed9ca826116da4 is 50, key is test_row_0/A:col10/1732246928526/Put/seqid=0 2024-11-22T03:42:09,442 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742288_1464 (size=12301) 2024-11-22T03:42:09,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-22T03:42:09,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(8855): Flush requested on 4a7ce0cfb17f4dcc3dd24b2156963795 2024-11-22T03:42:09,672 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795. as already flushing 2024-11-22T03:42:09,712 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:42:09,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40966 deadline: 1732246989699, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 2024-11-22T03:42:09,712 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:09,712 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:42:09,712 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:09,712 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:42:09,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40990 deadline: 1732246989703, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 2024-11-22T03:42:09,712 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:09,712 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:09,712 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:42:09,717 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:42:09,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40926 deadline: 1732246989704, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 2024-11-22T03:42:09,717 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:09,717 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:09,717 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:42:09,726 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/.tmp/B/34c1617ac80442279a56cf8de53a92a8 as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/B/34c1617ac80442279a56cf8de53a92a8 2024-11-22T03:42:09,731 INFO [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 4a7ce0cfb17f4dcc3dd24b2156963795/B of 4a7ce0cfb17f4dcc3dd24b2156963795 into 34c1617ac80442279a56cf8de53a92a8(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T03:42:09,731 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4a7ce0cfb17f4dcc3dd24b2156963795: 2024-11-22T03:42:09,731 INFO [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., storeName=4a7ce0cfb17f4dcc3dd24b2156963795/B, priority=12, startTime=1732246929303; duration=0sec 2024-11-22T03:42:09,731 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T03:42:09,731 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4a7ce0cfb17f4dcc3dd24b2156963795:B 2024-11-22T03:42:09,731 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-22T03:42:09,732 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49818 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-22T03:42:09,732 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HStore(1541): 4a7ce0cfb17f4dcc3dd24b2156963795/C is initiating minor compaction (all files) 2024-11-22T03:42:09,732 INFO [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4a7ce0cfb17f4dcc3dd24b2156963795/C in TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795. 2024-11-22T03:42:09,732 INFO [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/C/39c603a41ce04cfab5ed3a164df485ab, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/C/311ebb17261249979ffa70985e116e07, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/C/1a0a6472d57b4d0da4bea93e692ca07c, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/C/392b18f8879b47e3a54467440a77e8bb] into tmpdir=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/.tmp, totalSize=48.7 K 2024-11-22T03:42:09,733 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] compactions.Compactor(225): Compacting 39c603a41ce04cfab5ed3a164df485ab, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=269, earliestPutTs=1732246924288 2024-11-22T03:42:09,733 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] compactions.Compactor(225): Compacting 311ebb17261249979ffa70985e116e07, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=283, earliestPutTs=1732246926599 2024-11-22T03:42:09,733 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] compactions.Compactor(225): Compacting 1a0a6472d57b4d0da4bea93e692ca07c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=308, earliestPutTs=1732246926633 2024-11-22T03:42:09,734 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] compactions.Compactor(225): Compacting 392b18f8879b47e3a54467440a77e8bb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=320, earliestPutTs=1732246927811 2024-11-22T03:42:09,735 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/.tmp/A/bd2e8314cd6a432eaf51901bec5f9a4e as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/A/bd2e8314cd6a432eaf51901bec5f9a4e 2024-11-22T03:42:09,744 INFO [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 4a7ce0cfb17f4dcc3dd24b2156963795/A of 4a7ce0cfb17f4dcc3dd24b2156963795 into bd2e8314cd6a432eaf51901bec5f9a4e(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T03:42:09,744 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4a7ce0cfb17f4dcc3dd24b2156963795: 2024-11-22T03:42:09,745 INFO [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., storeName=4a7ce0cfb17f4dcc3dd24b2156963795/A, priority=12, startTime=1732246929303; duration=0sec 2024-11-22T03:42:09,745 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T03:42:09,745 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4a7ce0cfb17f4dcc3dd24b2156963795:A 2024-11-22T03:42:09,748 INFO [RS:0;c85114ed5096:42545-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4a7ce0cfb17f4dcc3dd24b2156963795#C#compaction#399 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T03:42:09,749 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/.tmp/C/8e4f9fea28654893a3f614d42c77d472 is 50, key is test_row_0/C:col10/1732246927817/Put/seqid=0 2024-11-22T03:42:09,756 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742289_1465 (size=13051) 2024-11-22T03:42:09,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-22T03:42:09,763 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/.tmp/C/8e4f9fea28654893a3f614d42c77d472 as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/C/8e4f9fea28654893a3f614d42c77d472 2024-11-22T03:42:09,767 INFO [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 4a7ce0cfb17f4dcc3dd24b2156963795/C of 4a7ce0cfb17f4dcc3dd24b2156963795 into 8e4f9fea28654893a3f614d42c77d472(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T03:42:09,767 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4a7ce0cfb17f4dcc3dd24b2156963795: 2024-11-22T03:42:09,767 INFO [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., storeName=4a7ce0cfb17f4dcc3dd24b2156963795/C, priority=12, startTime=1732246929303; duration=0sec 2024-11-22T03:42:09,767 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T03:42:09,767 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4a7ce0cfb17f4dcc3dd24b2156963795:C 2024-11-22T03:42:09,824 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:42:09,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40966 deadline: 1732246989818, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 2024-11-22T03:42:09,824 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:42:09,824 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:09,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40990 deadline: 1732246989818, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 2024-11-22T03:42:09,824 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:09,824 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:42:09,824 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:09,824 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:09,824 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:42:09,832 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:42:09,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40926 deadline: 1732246989828, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 2024-11-22T03:42:09,832 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:09,832 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:09,832 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:42:09,843 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=344 (bloomFilter=true), to=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/.tmp/A/442b6a0715154852b2ed9ca826116da4 2024-11-22T03:42:09,848 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/.tmp/B/889f23f7f1f945c9acbcae921888c97a is 50, key is test_row_0/B:col10/1732246928526/Put/seqid=0 2024-11-22T03:42:09,852 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742290_1466 (size=12301) 2024-11-22T03:42:10,034 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:42:10,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40966 deadline: 1732246990028, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 2024-11-22T03:42:10,034 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:42:10,035 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:10,035 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:10,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40990 deadline: 1732246990028, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 2024-11-22T03:42:10,035 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:42:10,035 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:10,035 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:10,035 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:42:10,044 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:42:10,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40926 deadline: 1732246990039, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 2024-11-22T03:42:10,044 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:10,044 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:10,044 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:42:10,252 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=344 (bloomFilter=true), to=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/.tmp/B/889f23f7f1f945c9acbcae921888c97a 2024-11-22T03:42:10,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/.tmp/C/150142ae2e7942f8bb86fbab1f5f3696 is 50, key is test_row_0/C:col10/1732246928526/Put/seqid=0 2024-11-22T03:42:10,265 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742291_1467 (size=12301) 2024-11-22T03:42:10,266 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=344 (bloomFilter=true), to=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/.tmp/C/150142ae2e7942f8bb86fbab1f5f3696 2024-11-22T03:42:10,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-22T03:42:10,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/.tmp/A/442b6a0715154852b2ed9ca826116da4 as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/A/442b6a0715154852b2ed9ca826116da4 2024-11-22T03:42:10,273 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/A/442b6a0715154852b2ed9ca826116da4, entries=150, sequenceid=344, filesize=12.0 K 2024-11-22T03:42:10,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/.tmp/B/889f23f7f1f945c9acbcae921888c97a as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/B/889f23f7f1f945c9acbcae921888c97a 2024-11-22T03:42:10,279 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/B/889f23f7f1f945c9acbcae921888c97a, entries=150, sequenceid=344, filesize=12.0 K 2024-11-22T03:42:10,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/.tmp/C/150142ae2e7942f8bb86fbab1f5f3696 as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/C/150142ae2e7942f8bb86fbab1f5f3696 2024-11-22T03:42:10,283 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/C/150142ae2e7942f8bb86fbab1f5f3696, entries=150, sequenceid=344, filesize=12.0 K 2024-11-22T03:42:10,284 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for 4a7ce0cfb17f4dcc3dd24b2156963795 in 849ms, sequenceid=344, compaction requested=false 2024-11-22T03:42:10,284 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2603): Flush status journal for 4a7ce0cfb17f4dcc3dd24b2156963795: 2024-11-22T03:42:10,284 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795. 2024-11-22T03:42:10,284 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=136 2024-11-22T03:42:10,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42627 {}] master.HMaster(4169): Remote procedure done, pid=136 2024-11-22T03:42:10,288 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=136, resume processing ppid=135 2024-11-22T03:42:10,289 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=136, ppid=135, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1540 sec 2024-11-22T03:42:10,290 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=135, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees in 1.1600 sec 2024-11-22T03:42:10,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(8855): Flush requested on 4a7ce0cfb17f4dcc3dd24b2156963795 2024-11-22T03:42:10,344 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-22T03:42:10,344 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4a7ce0cfb17f4dcc3dd24b2156963795 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-22T03:42:10,350 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/.tmp/A/7fec46701cd54a1d972f79a50b432142 is 50, key is test_row_0/A:col10/1732246929698/Put/seqid=0 2024-11-22T03:42:10,354 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742292_1468 (size=17181) 2024-11-22T03:42:10,399 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:42:10,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40966 deadline: 1732246990391, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 2024-11-22T03:42:10,399 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:42:10,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40990 deadline: 1732246990393, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 2024-11-22T03:42:10,399 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:10,399 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:10,399 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:42:10,399 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:10,399 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:10,400 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:42:10,407 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:42:10,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40926 deadline: 1732246990400, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 2024-11-22T03:42:10,407 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:10,407 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:10,407 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:42:10,512 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:42:10,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40990 deadline: 1732246990509, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 2024-11-22T03:42:10,512 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:42:10,512 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:10,513 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:10,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40966 deadline: 1732246990509, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 2024-11-22T03:42:10,513 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:42:10,513 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:10,513 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:42:10,513 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:10,513 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:42:10,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40926 deadline: 1732246990509, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 2024-11-22T03:42:10,513 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:10,513 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:10,513 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:42:10,724 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:42:10,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40990 deadline: 1732246990718, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 2024-11-22T03:42:10,724 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:10,724 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:10,724 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:42:10,724 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:42:10,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40966 deadline: 1732246990718, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 2024-11-22T03:42:10,725 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:10,725 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:10,725 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:42:10,728 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:42:10,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40926 deadline: 1732246990719, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 2024-11-22T03:42:10,728 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:10,728 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:10,728 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:42:10,755 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=361 (bloomFilter=true), to=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/.tmp/A/7fec46701cd54a1d972f79a50b432142 2024-11-22T03:42:10,760 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/.tmp/B/47cd5a9870994f05adbbd14fef32693d is 50, key is test_row_0/B:col10/1732246929698/Put/seqid=0 2024-11-22T03:42:10,763 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742293_1469 (size=12301) 2024-11-22T03:42:10,793 INFO [Thread-1816 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1816. 2024-11-22T03:42:10,793 DEBUG [Thread-1816 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-22T03:42:10,793 DEBUG [Thread-1816 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T03:42:10,793 DEBUG [Thread-1816 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T03:42:10,793 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-22T03:42:10,794 INFO [Thread-1812 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1812. 2024-11-22T03:42:10,794 DEBUG [Thread-1812 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-22T03:42:10,794 DEBUG [Thread-1812 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T03:42:10,794 DEBUG [Thread-1812 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T03:42:10,794 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-22T03:42:10,798 INFO [Thread-1820 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1820. 2024-11-22T03:42:10,799 DEBUG [Thread-1820 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-22T03:42:10,799 DEBUG [Thread-1820 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T03:42:10,799 DEBUG [Thread-1820 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T03:42:10,799 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-22T03:42:10,802 INFO [Thread-1814 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1814. 2024-11-22T03:42:10,802 INFO [Thread-1818 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1818. 2024-11-22T03:42:10,802 DEBUG [Thread-1814 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-22T03:42:10,802 DEBUG [Thread-1814 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T03:42:10,802 DEBUG [Thread-1818 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-22T03:42:10,802 DEBUG [Thread-1818 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T03:42:10,802 DEBUG [Thread-1814 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T03:42:10,802 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-22T03:42:10,802 DEBUG [Thread-1818 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T03:42:10,803 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-22T03:42:11,030 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:42:11,031 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:42:11,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40990 deadline: 1732246991030, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 2024-11-22T03:42:11,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40966 deadline: 1732246991030, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 2024-11-22T03:42:11,032 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:11,032 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:11,032 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:11,032 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:11,032 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:42:11,032 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:42:11,039 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:42:11,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40926 deadline: 1732246991039, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 2024-11-22T03:42:11,040 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:11,040 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:11,040 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:42:11,049 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:42:11,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41022 deadline: 1732246991048, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 2024-11-22T03:42:11,050 WARN [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRpcRetryingCaller(177): Call to c85114ed5096,42545,1732246786482 for 'test_row_0' in 4a7ce0cfb17f4dcc3dd24b2156963795 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8198 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-22T03:42:11,050 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:11,050 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:11,051 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:42:11,069 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:42:11,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40962 deadline: 1732246991069, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 2024-11-22T03:42:11,070 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to c85114ed5096,42545,1732246786482 for 'test_row_0' in 4a7ce0cfb17f4dcc3dd24b2156963795 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8221 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-22T03:42:11,071 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:11,071 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:11,071 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:42:11,165 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=361 (bloomFilter=true), to=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/.tmp/B/47cd5a9870994f05adbbd14fef32693d 2024-11-22T03:42:11,177 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/.tmp/C/1da3fb94b44f4618b7b755006f46f028 is 50, key is test_row_0/C:col10/1732246929698/Put/seqid=0 2024-11-22T03:42:11,181 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742294_1470 (size=12301) 2024-11-22T03:42:11,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-22T03:42:11,279 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-22T03:42:11,540 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:42:11,540 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:42:11,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40990 deadline: 1732246991539, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 2024-11-22T03:42:11,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40966 deadline: 1732246991539, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 2024-11-22T03:42:11,541 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:11,541 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:11,541 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:11,541 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:11,541 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:42:11,541 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:42:11,549 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:42:11,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40926 deadline: 1732246991549, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 2024-11-22T03:42:11,550 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 , the old value is region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:11,550 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:11,550 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., hostname=c85114ed5096,42545,1732246786482, seqNum=2 because the exception is null or not the one we care about 2024-11-22T03:42:11,582 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=361 (bloomFilter=true), to=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/.tmp/C/1da3fb94b44f4618b7b755006f46f028 2024-11-22T03:42:11,592 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/.tmp/A/7fec46701cd54a1d972f79a50b432142 as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/A/7fec46701cd54a1d972f79a50b432142 2024-11-22T03:42:11,599 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/A/7fec46701cd54a1d972f79a50b432142, entries=250, sequenceid=361, filesize=16.8 K 2024-11-22T03:42:11,600 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/.tmp/B/47cd5a9870994f05adbbd14fef32693d as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/B/47cd5a9870994f05adbbd14fef32693d 2024-11-22T03:42:11,604 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/B/47cd5a9870994f05adbbd14fef32693d, entries=150, sequenceid=361, filesize=12.0 K 2024-11-22T03:42:11,605 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/.tmp/C/1da3fb94b44f4618b7b755006f46f028 as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/C/1da3fb94b44f4618b7b755006f46f028 2024-11-22T03:42:11,608 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/C/1da3fb94b44f4618b7b755006f46f028, entries=150, sequenceid=361, filesize=12.0 K 2024-11-22T03:42:11,608 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for 4a7ce0cfb17f4dcc3dd24b2156963795 in 1264ms, sequenceid=361, compaction requested=true 2024-11-22T03:42:11,608 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4a7ce0cfb17f4dcc3dd24b2156963795: 2024-11-22T03:42:11,609 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4a7ce0cfb17f4dcc3dd24b2156963795:A, priority=-2147483648, current under compaction store size is 1 2024-11-22T03:42:11,609 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T03:42:11,609 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T03:42:11,609 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4a7ce0cfb17f4dcc3dd24b2156963795:B, priority=-2147483648, current under compaction store size is 2 2024-11-22T03:42:11,609 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T03:42:11,609 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4a7ce0cfb17f4dcc3dd24b2156963795:C, priority=-2147483648, current under compaction store size is 3 2024-11-22T03:42:11,609 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T03:42:11,609 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T03:42:11,610 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 42533 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T03:42:11,610 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T03:42:11,610 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.HStore(1541): 4a7ce0cfb17f4dcc3dd24b2156963795/A is initiating minor compaction (all files) 2024-11-22T03:42:11,610 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HStore(1541): 4a7ce0cfb17f4dcc3dd24b2156963795/B is initiating minor compaction (all files) 2024-11-22T03:42:11,610 INFO [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4a7ce0cfb17f4dcc3dd24b2156963795/A in TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795. 2024-11-22T03:42:11,610 INFO [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4a7ce0cfb17f4dcc3dd24b2156963795/B in TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795. 2024-11-22T03:42:11,610 INFO [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/A/bd2e8314cd6a432eaf51901bec5f9a4e, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/A/442b6a0715154852b2ed9ca826116da4, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/A/7fec46701cd54a1d972f79a50b432142] into tmpdir=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/.tmp, totalSize=41.5 K 2024-11-22T03:42:11,610 INFO [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/B/34c1617ac80442279a56cf8de53a92a8, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/B/889f23f7f1f945c9acbcae921888c97a, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/B/47cd5a9870994f05adbbd14fef32693d] into tmpdir=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/.tmp, totalSize=36.8 K 2024-11-22T03:42:11,610 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] compactions.Compactor(225): Compacting bd2e8314cd6a432eaf51901bec5f9a4e, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=320, earliestPutTs=1732246927811 2024-11-22T03:42:11,610 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] compactions.Compactor(225): Compacting 34c1617ac80442279a56cf8de53a92a8, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=320, earliestPutTs=1732246927811 2024-11-22T03:42:11,610 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] compactions.Compactor(225): Compacting 442b6a0715154852b2ed9ca826116da4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=344, earliestPutTs=1732246928514 2024-11-22T03:42:11,611 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] compactions.Compactor(225): Compacting 889f23f7f1f945c9acbcae921888c97a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=344, earliestPutTs=1732246928514 2024-11-22T03:42:11,611 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7fec46701cd54a1d972f79a50b432142, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=361, earliestPutTs=1732246929697 2024-11-22T03:42:11,611 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] compactions.Compactor(225): Compacting 47cd5a9870994f05adbbd14fef32693d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=361, earliestPutTs=1732246929698 2024-11-22T03:42:11,621 INFO [RS:0;c85114ed5096:42545-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4a7ce0cfb17f4dcc3dd24b2156963795#B#compaction#405 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T03:42:11,621 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/.tmp/B/3e5ecaa50d4b45ffb3d027dc6016b2fd is 50, key is test_row_0/B:col10/1732246929698/Put/seqid=0 2024-11-22T03:42:11,625 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742295_1471 (size=13153) 2024-11-22T03:42:11,644 INFO [RS:0;c85114ed5096:42545-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4a7ce0cfb17f4dcc3dd24b2156963795#A#compaction#406 average throughput is 0.30 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T03:42:11,644 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/.tmp/A/4669d86ae53c49d4b3b4cf36483e9448 is 50, key is test_row_0/A:col10/1732246929698/Put/seqid=0 2024-11-22T03:42:11,647 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742296_1472 (size=13153) 2024-11-22T03:42:12,029 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/.tmp/B/3e5ecaa50d4b45ffb3d027dc6016b2fd as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/B/3e5ecaa50d4b45ffb3d027dc6016b2fd 2024-11-22T03:42:12,032 INFO [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4a7ce0cfb17f4dcc3dd24b2156963795/B of 4a7ce0cfb17f4dcc3dd24b2156963795 into 3e5ecaa50d4b45ffb3d027dc6016b2fd(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T03:42:12,032 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4a7ce0cfb17f4dcc3dd24b2156963795: 2024-11-22T03:42:12,032 INFO [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., storeName=4a7ce0cfb17f4dcc3dd24b2156963795/B, priority=13, startTime=1732246931609; duration=0sec 2024-11-22T03:42:12,032 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T03:42:12,033 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4a7ce0cfb17f4dcc3dd24b2156963795:B 2024-11-22T03:42:12,033 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T03:42:12,033 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T03:42:12,033 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HStore(1541): 4a7ce0cfb17f4dcc3dd24b2156963795/C is initiating minor compaction (all files) 2024-11-22T03:42:12,033 INFO [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4a7ce0cfb17f4dcc3dd24b2156963795/C in TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795. 2024-11-22T03:42:12,033 INFO [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/C/8e4f9fea28654893a3f614d42c77d472, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/C/150142ae2e7942f8bb86fbab1f5f3696, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/C/1da3fb94b44f4618b7b755006f46f028] into tmpdir=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/.tmp, totalSize=36.8 K 2024-11-22T03:42:12,034 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] compactions.Compactor(225): Compacting 8e4f9fea28654893a3f614d42c77d472, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=320, earliestPutTs=1732246927811 2024-11-22T03:42:12,034 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] compactions.Compactor(225): Compacting 150142ae2e7942f8bb86fbab1f5f3696, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=344, earliestPutTs=1732246928514 2024-11-22T03:42:12,034 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] compactions.Compactor(225): Compacting 1da3fb94b44f4618b7b755006f46f028, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=361, earliestPutTs=1732246929698 2024-11-22T03:42:12,040 INFO [RS:0;c85114ed5096:42545-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4a7ce0cfb17f4dcc3dd24b2156963795#C#compaction#407 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T03:42:12,040 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/.tmp/C/acbb92838a044d72acf1d49e79a42cc9 is 50, key is test_row_0/C:col10/1732246929698/Put/seqid=0 2024-11-22T03:42:12,043 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742297_1473 (size=13153) 2024-11-22T03:42:12,051 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/.tmp/A/4669d86ae53c49d4b3b4cf36483e9448 as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/A/4669d86ae53c49d4b3b4cf36483e9448 2024-11-22T03:42:12,054 INFO [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4a7ce0cfb17f4dcc3dd24b2156963795/A of 4a7ce0cfb17f4dcc3dd24b2156963795 into 4669d86ae53c49d4b3b4cf36483e9448(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T03:42:12,054 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4a7ce0cfb17f4dcc3dd24b2156963795: 2024-11-22T03:42:12,054 INFO [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., storeName=4a7ce0cfb17f4dcc3dd24b2156963795/A, priority=13, startTime=1732246931608; duration=0sec 2024-11-22T03:42:12,054 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T03:42:12,054 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4a7ce0cfb17f4dcc3dd24b2156963795:A 2024-11-22T03:42:12,447 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/.tmp/C/acbb92838a044d72acf1d49e79a42cc9 as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/C/acbb92838a044d72acf1d49e79a42cc9 2024-11-22T03:42:12,450 INFO [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4a7ce0cfb17f4dcc3dd24b2156963795/C of 4a7ce0cfb17f4dcc3dd24b2156963795 into acbb92838a044d72acf1d49e79a42cc9(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T03:42:12,450 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4a7ce0cfb17f4dcc3dd24b2156963795: 2024-11-22T03:42:12,450 INFO [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., storeName=4a7ce0cfb17f4dcc3dd24b2156963795/C, priority=13, startTime=1732246931609; duration=0sec 2024-11-22T03:42:12,450 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T03:42:12,450 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4a7ce0cfb17f4dcc3dd24b2156963795:C 2024-11-22T03:42:12,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(8855): Flush requested on 4a7ce0cfb17f4dcc3dd24b2156963795 2024-11-22T03:42:12,550 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-22T03:42:12,550 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4a7ce0cfb17f4dcc3dd24b2156963795 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-22T03:42:12,550 INFO [Thread-1807 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1807. 2024-11-22T03:42:12,551 DEBUG [Thread-1807 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-22T03:42:12,551 INFO [Thread-1801 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1801. 2024-11-22T03:42:12,551 DEBUG [Thread-1807 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T03:42:12,551 DEBUG [Thread-1801 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-22T03:42:12,551 DEBUG [Thread-1801 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T03:42:12,551 DEBUG [Thread-1807 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T03:42:12,551 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-22T03:42:12,551 DEBUG [Thread-1801 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T03:42:12,551 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-22T03:42:12,554 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/.tmp/A/932040e0a0f84b3da137ea5ac99f87b9 is 50, key is test_row_0/A:col10/1732246930392/Put/seqid=0 2024-11-22T03:42:12,557 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742298_1474 (size=12301) 2024-11-22T03:42:12,560 INFO [Thread-1805 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1805. 2024-11-22T03:42:12,560 DEBUG [Thread-1805 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-22T03:42:12,560 DEBUG [Thread-1805 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T03:42:12,560 DEBUG [Thread-1805 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T03:42:12,560 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-22T03:42:12,957 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=388 (bloomFilter=true), to=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/.tmp/A/932040e0a0f84b3da137ea5ac99f87b9 2024-11-22T03:42:12,963 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/.tmp/B/c8a2fdb78834451c844cb8cf65b14f52 is 50, key is test_row_0/B:col10/1732246930392/Put/seqid=0 2024-11-22T03:42:12,966 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742299_1475 (size=12301) 2024-11-22T03:42:13,366 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=388 (bloomFilter=true), to=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/.tmp/B/c8a2fdb78834451c844cb8cf65b14f52 2024-11-22T03:42:13,373 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/.tmp/C/3a08480090f2453882beafbae5c06c0c is 50, key is test_row_0/C:col10/1732246930392/Put/seqid=0 2024-11-22T03:42:13,377 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742300_1476 (size=12301) 2024-11-22T03:42:13,779 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=388 (bloomFilter=true), to=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/.tmp/C/3a08480090f2453882beafbae5c06c0c 2024-11-22T03:42:13,790 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/.tmp/A/932040e0a0f84b3da137ea5ac99f87b9 as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/A/932040e0a0f84b3da137ea5ac99f87b9 2024-11-22T03:42:13,793 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/A/932040e0a0f84b3da137ea5ac99f87b9, entries=150, sequenceid=388, filesize=12.0 K 2024-11-22T03:42:13,794 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/.tmp/B/c8a2fdb78834451c844cb8cf65b14f52 as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/B/c8a2fdb78834451c844cb8cf65b14f52 2024-11-22T03:42:13,797 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/B/c8a2fdb78834451c844cb8cf65b14f52, entries=150, sequenceid=388, filesize=12.0 K 2024-11-22T03:42:13,798 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/.tmp/C/3a08480090f2453882beafbae5c06c0c as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/C/3a08480090f2453882beafbae5c06c0c 2024-11-22T03:42:13,801 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/C/3a08480090f2453882beafbae5c06c0c, entries=150, sequenceid=388, filesize=12.0 K 2024-11-22T03:42:13,802 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=6.71 KB/6870 for 4a7ce0cfb17f4dcc3dd24b2156963795 in 1252ms, sequenceid=388, compaction requested=false 2024-11-22T03:42:13,802 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4a7ce0cfb17f4dcc3dd24b2156963795: 2024-11-22T03:42:14,823 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-22T03:42:21,080 INFO [Thread-1809 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1809. 2024-11-22T03:42:21,080 DEBUG [Thread-1809 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-22T03:42:21,081 DEBUG [Thread-1809 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T03:42:21,081 DEBUG [Thread-1809 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T03:42:21,081 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-22T03:42:21,130 INFO [Thread-1803 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1803. 2024-11-22T03:42:21,130 DEBUG [Thread-1803 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-22T03:42:21,130 DEBUG [Thread-1803 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T03:42:21,130 DEBUG [Thread-1803 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T03:42:21,130 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-22T03:42:21,130 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-22T03:42:21,130 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 82 2024-11-22T03:42:21,130 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 31 2024-11-22T03:42:21,130 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 85 2024-11-22T03:42:21,130 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 85 2024-11-22T03:42:21,130 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 26 2024-11-22T03:42:21,130 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-22T03:42:21,130 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-22T03:42:21,130 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1583 2024-11-22T03:42:21,130 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4739 rows 2024-11-22T03:42:21,130 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1590 2024-11-22T03:42:21,130 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4770 rows 2024-11-22T03:42:21,130 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1584 2024-11-22T03:42:21,130 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4742 rows 2024-11-22T03:42:21,130 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1585 2024-11-22T03:42:21,130 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4749 rows 2024-11-22T03:42:21,130 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1586 2024-11-22T03:42:21,130 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4758 rows 2024-11-22T03:42:21,130 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-22T03:42:21,130 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testScanAtomicity(AcidGuaranteesTestBase.java:112) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-22T03:42:21,131 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T03:42:21,131 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T03:42:21,131 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-22T03:42:21,131 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42627 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-22T03:42:21,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42627 {}] procedure2.ProcedureExecutor(1139): Stored pid=137, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-22T03:42:21,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-22T03:42:21,134 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732246941134"}]},"ts":"1732246941134"} 2024-11-22T03:42:21,135 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-22T03:42:21,135 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-22T03:42:21,136 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=138, ppid=137, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-22T03:42:21,137 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=139, ppid=138, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=4a7ce0cfb17f4dcc3dd24b2156963795, UNASSIGN}] 2024-11-22T03:42:21,137 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=139, ppid=138, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=4a7ce0cfb17f4dcc3dd24b2156963795, UNASSIGN 2024-11-22T03:42:21,138 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=139 updating hbase:meta row=4a7ce0cfb17f4dcc3dd24b2156963795, regionState=CLOSING, regionLocation=c85114ed5096,42545,1732246786482 2024-11-22T03:42:21,139 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=139, ppid=138, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=4a7ce0cfb17f4dcc3dd24b2156963795, UNASSIGN because future has completed 2024-11-22T03:42:21,139 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-22T03:42:21,139 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=140, ppid=139, state=RUNNABLE, hasLock=false; CloseRegionProcedure 4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482}] 2024-11-22T03:42:21,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-22T03:42:21,291 INFO [RS_CLOSE_REGION-regionserver/c85114ed5096:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] handler.UnassignRegionHandler(122): Close 4a7ce0cfb17f4dcc3dd24b2156963795 2024-11-22T03:42:21,291 DEBUG [RS_CLOSE_REGION-regionserver/c85114ed5096:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-22T03:42:21,292 DEBUG [RS_CLOSE_REGION-regionserver/c85114ed5096:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegion(1722): Closing 4a7ce0cfb17f4dcc3dd24b2156963795, disabling compactions & flushes 2024-11-22T03:42:21,292 INFO [RS_CLOSE_REGION-regionserver/c85114ed5096:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795. 2024-11-22T03:42:21,292 DEBUG [RS_CLOSE_REGION-regionserver/c85114ed5096:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795. 2024-11-22T03:42:21,292 DEBUG [RS_CLOSE_REGION-regionserver/c85114ed5096:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795. after waiting 0 ms 2024-11-22T03:42:21,292 DEBUG [RS_CLOSE_REGION-regionserver/c85114ed5096:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795. 2024-11-22T03:42:21,292 INFO [RS_CLOSE_REGION-regionserver/c85114ed5096:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegion(2902): Flushing 4a7ce0cfb17f4dcc3dd24b2156963795 3/3 column families, dataSize=20.13 KB heapSize=53.48 KB 2024-11-22T03:42:21,296 DEBUG [RS_CLOSE_REGION-regionserver/c85114ed5096:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/.tmp/A/34ab25aeaa6744ac87ee19623d03ec17 is 50, key is test_row_0/A:col10/1732246941128/Put/seqid=0 2024-11-22T03:42:21,299 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742301_1477 (size=7415) 2024-11-22T03:42:21,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-22T03:42:21,700 INFO [RS_CLOSE_REGION-regionserver/c85114ed5096:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=394 (bloomFilter=true), to=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/.tmp/A/34ab25aeaa6744ac87ee19623d03ec17 2024-11-22T03:42:21,714 DEBUG [RS_CLOSE_REGION-regionserver/c85114ed5096:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/.tmp/B/6210bd8e2a164e3ba3e20ac88744c464 is 50, key is test_row_0/B:col10/1732246941128/Put/seqid=0 2024-11-22T03:42:21,718 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742302_1478 (size=7415) 2024-11-22T03:42:21,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-22T03:42:22,119 INFO [RS_CLOSE_REGION-regionserver/c85114ed5096:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=394 (bloomFilter=true), to=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/.tmp/B/6210bd8e2a164e3ba3e20ac88744c464 2024-11-22T03:42:22,132 DEBUG [RS_CLOSE_REGION-regionserver/c85114ed5096:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/.tmp/C/a4a5d2874e2e4846a2f5d15a5968a691 is 50, key is test_row_0/C:col10/1732246941128/Put/seqid=0 2024-11-22T03:42:22,135 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742303_1479 (size=7415) 2024-11-22T03:42:22,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-22T03:42:22,536 INFO [RS_CLOSE_REGION-regionserver/c85114ed5096:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=394 (bloomFilter=true), to=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/.tmp/C/a4a5d2874e2e4846a2f5d15a5968a691 2024-11-22T03:42:22,540 DEBUG [RS_CLOSE_REGION-regionserver/c85114ed5096:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/.tmp/A/34ab25aeaa6744ac87ee19623d03ec17 as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/A/34ab25aeaa6744ac87ee19623d03ec17 2024-11-22T03:42:22,543 INFO [RS_CLOSE_REGION-regionserver/c85114ed5096:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/A/34ab25aeaa6744ac87ee19623d03ec17, entries=50, sequenceid=394, filesize=7.2 K 2024-11-22T03:42:22,543 DEBUG [RS_CLOSE_REGION-regionserver/c85114ed5096:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/.tmp/B/6210bd8e2a164e3ba3e20ac88744c464 as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/B/6210bd8e2a164e3ba3e20ac88744c464 2024-11-22T03:42:22,546 INFO [RS_CLOSE_REGION-regionserver/c85114ed5096:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/B/6210bd8e2a164e3ba3e20ac88744c464, entries=50, sequenceid=394, filesize=7.2 K 2024-11-22T03:42:22,547 DEBUG [RS_CLOSE_REGION-regionserver/c85114ed5096:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/.tmp/C/a4a5d2874e2e4846a2f5d15a5968a691 as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/C/a4a5d2874e2e4846a2f5d15a5968a691 2024-11-22T03:42:22,549 INFO [RS_CLOSE_REGION-regionserver/c85114ed5096:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/C/a4a5d2874e2e4846a2f5d15a5968a691, entries=50, sequenceid=394, filesize=7.2 K 2024-11-22T03:42:22,550 INFO [RS_CLOSE_REGION-regionserver/c85114ed5096:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegion(3140): Finished flush of dataSize ~20.13 KB/20610, heapSize ~53.44 KB/54720, currentSize=0 B/0 for 4a7ce0cfb17f4dcc3dd24b2156963795 in 1258ms, sequenceid=394, compaction requested=true 2024-11-22T03:42:22,550 DEBUG [StoreCloser-TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/A/ac88f1cb9e334b71884220ec66c56be9, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/A/22690dbf6bed459ca430a14060adfafe, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/A/71277d86252c497a98179237e4977b01, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/A/d54a9efc230f472e9418bc21f61f0f69, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/A/b8f71668dd8c40b39cd0500452b61a80, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/A/98f76382f1b64743ba793f18db616218, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/A/94b11a7be9aa448885581b203cbb3c99, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/A/8d1d6eaa02cc4493ac2e1c802f395cb4, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/A/7310966b824440a9bb996c72dc5b5e1a, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/A/cf322b78952c409f8183ae1743d6022a, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/A/31722d9bf9d74620b78e43a09511105d, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/A/bc05629301bf427595f60e80b6b07a2e, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/A/892770ca81db42de8adbb8f81e60a526, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/A/8e923d4205144abea3d6f5e110f7961b, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/A/44ab2d6996b046d493a4e6f63b3d09cd, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/A/7731601f62e9420b8eb5bf7e2b49979e, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/A/6ec78fef2e624aa5af9b251465f1983f, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/A/a612e4acab9e45cca2fb4976211a51bb, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/A/b0ff8e7b62d44bcebcec077f4aeeea18, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/A/2e804fefc6474369a9c4f8fa9d19d814, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/A/6cad706e30104a84b7a229e70c4c778d, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/A/ef2a06fcf77541979a97da3f08167b8c, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/A/bd2e8314cd6a432eaf51901bec5f9a4e, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/A/442b6a0715154852b2ed9ca826116da4, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/A/7fec46701cd54a1d972f79a50b432142] to archive 2024-11-22T03:42:22,551 DEBUG [StoreCloser-TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-22T03:42:22,552 DEBUG [StoreCloser-TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/A/ac88f1cb9e334b71884220ec66c56be9 to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/A/ac88f1cb9e334b71884220ec66c56be9 2024-11-22T03:42:22,553 DEBUG [StoreCloser-TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/A/22690dbf6bed459ca430a14060adfafe to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/A/22690dbf6bed459ca430a14060adfafe 2024-11-22T03:42:22,553 DEBUG [StoreCloser-TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/A/71277d86252c497a98179237e4977b01 to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/A/71277d86252c497a98179237e4977b01 2024-11-22T03:42:22,554 DEBUG [StoreCloser-TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/A/d54a9efc230f472e9418bc21f61f0f69 to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/A/d54a9efc230f472e9418bc21f61f0f69 2024-11-22T03:42:22,555 DEBUG [StoreCloser-TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/A/b8f71668dd8c40b39cd0500452b61a80 to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/A/b8f71668dd8c40b39cd0500452b61a80 2024-11-22T03:42:22,556 DEBUG [StoreCloser-TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/A/98f76382f1b64743ba793f18db616218 to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/A/98f76382f1b64743ba793f18db616218 2024-11-22T03:42:22,556 DEBUG [StoreCloser-TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/A/94b11a7be9aa448885581b203cbb3c99 to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/A/94b11a7be9aa448885581b203cbb3c99 2024-11-22T03:42:22,557 DEBUG [StoreCloser-TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/A/8d1d6eaa02cc4493ac2e1c802f395cb4 to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/A/8d1d6eaa02cc4493ac2e1c802f395cb4 2024-11-22T03:42:22,558 DEBUG [StoreCloser-TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/A/7310966b824440a9bb996c72dc5b5e1a to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/A/7310966b824440a9bb996c72dc5b5e1a 2024-11-22T03:42:22,558 DEBUG [StoreCloser-TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/A/cf322b78952c409f8183ae1743d6022a to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/A/cf322b78952c409f8183ae1743d6022a 2024-11-22T03:42:22,559 DEBUG [StoreCloser-TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/A/31722d9bf9d74620b78e43a09511105d to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/A/31722d9bf9d74620b78e43a09511105d 2024-11-22T03:42:22,560 DEBUG [StoreCloser-TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/A/bc05629301bf427595f60e80b6b07a2e to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/A/bc05629301bf427595f60e80b6b07a2e 2024-11-22T03:42:22,561 DEBUG [StoreCloser-TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/A/892770ca81db42de8adbb8f81e60a526 to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/A/892770ca81db42de8adbb8f81e60a526 2024-11-22T03:42:22,562 DEBUG [StoreCloser-TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/A/8e923d4205144abea3d6f5e110f7961b to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/A/8e923d4205144abea3d6f5e110f7961b 2024-11-22T03:42:22,563 DEBUG [StoreCloser-TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/A/44ab2d6996b046d493a4e6f63b3d09cd to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/A/44ab2d6996b046d493a4e6f63b3d09cd 2024-11-22T03:42:22,563 DEBUG [StoreCloser-TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/A/7731601f62e9420b8eb5bf7e2b49979e to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/A/7731601f62e9420b8eb5bf7e2b49979e 2024-11-22T03:42:22,564 DEBUG [StoreCloser-TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/A/6ec78fef2e624aa5af9b251465f1983f to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/A/6ec78fef2e624aa5af9b251465f1983f 2024-11-22T03:42:22,565 DEBUG [StoreCloser-TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/A/a612e4acab9e45cca2fb4976211a51bb to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/A/a612e4acab9e45cca2fb4976211a51bb 2024-11-22T03:42:22,565 DEBUG [StoreCloser-TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/A/b0ff8e7b62d44bcebcec077f4aeeea18 to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/A/b0ff8e7b62d44bcebcec077f4aeeea18 2024-11-22T03:42:22,566 DEBUG [StoreCloser-TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/A/2e804fefc6474369a9c4f8fa9d19d814 to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/A/2e804fefc6474369a9c4f8fa9d19d814 2024-11-22T03:42:22,567 DEBUG [StoreCloser-TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/A/6cad706e30104a84b7a229e70c4c778d to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/A/6cad706e30104a84b7a229e70c4c778d 2024-11-22T03:42:22,568 DEBUG [StoreCloser-TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/A/ef2a06fcf77541979a97da3f08167b8c to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/A/ef2a06fcf77541979a97da3f08167b8c 2024-11-22T03:42:22,568 DEBUG [StoreCloser-TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/A/bd2e8314cd6a432eaf51901bec5f9a4e to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/A/bd2e8314cd6a432eaf51901bec5f9a4e 2024-11-22T03:42:22,569 DEBUG [StoreCloser-TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/A/442b6a0715154852b2ed9ca826116da4 to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/A/442b6a0715154852b2ed9ca826116da4 2024-11-22T03:42:22,570 DEBUG [StoreCloser-TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/A/7fec46701cd54a1d972f79a50b432142 to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/A/7fec46701cd54a1d972f79a50b432142 2024-11-22T03:42:22,571 DEBUG [StoreCloser-TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/B/4bb69e136b8c47268a9c8a395c02484b, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/B/a74e1c34d9f941db99f8b7cc570fd51c, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/B/1d942119e672441586705152a7d45a23, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/B/f283a1d9d4e7406fb7aeaffbe3e43f82, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/B/a55c23f912144f2c97cec3b2f8acf963, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/B/ac98440d556545e0ae9f8aa814757c9d, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/B/829066b589504d938b2fb6ac37a879b1, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/B/9390ed4698b340348b6f6a8156f9b8da, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/B/26dc1f560f78442da4005f4162884cfe, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/B/3bb2b7f488f54c5595ce617df05a1b7d, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/B/554b916013a348b2b1739e3d0237de5f, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/B/031726fec90d4c01ae917ca441ec14d5, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/B/6b664097f71749ed85806a4b6afcb230, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/B/96ff445498f445dc8dcd512fb3da5c27, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/B/07d6e42780ed433d9e8b80bb72c2d006, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/B/9bcaa526e82641cab5ce90d52f648996, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/B/c9723ccbbfe24071a0dc29c27912627d, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/B/9b17341d9a7c4c808ca45da794e72e12, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/B/985850e0381e47ad878559d3c83cf4aa, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/B/9f40ad9c40204d738e211c2f0eacf91e, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/B/383df294c68848c6a474c4249586c1a2, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/B/34c1617ac80442279a56cf8de53a92a8, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/B/2cecc979d6e748f497e69eb98b8e5e90, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/B/889f23f7f1f945c9acbcae921888c97a, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/B/47cd5a9870994f05adbbd14fef32693d] to archive 2024-11-22T03:42:22,572 DEBUG [StoreCloser-TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-22T03:42:22,573 DEBUG [StoreCloser-TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/B/4bb69e136b8c47268a9c8a395c02484b to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/B/4bb69e136b8c47268a9c8a395c02484b 2024-11-22T03:42:22,574 DEBUG [StoreCloser-TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/B/a74e1c34d9f941db99f8b7cc570fd51c to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/B/a74e1c34d9f941db99f8b7cc570fd51c 2024-11-22T03:42:22,575 DEBUG [StoreCloser-TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/B/1d942119e672441586705152a7d45a23 to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/B/1d942119e672441586705152a7d45a23 2024-11-22T03:42:22,576 DEBUG [StoreCloser-TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/B/f283a1d9d4e7406fb7aeaffbe3e43f82 to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/B/f283a1d9d4e7406fb7aeaffbe3e43f82 2024-11-22T03:42:22,576 DEBUG [StoreCloser-TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/B/a55c23f912144f2c97cec3b2f8acf963 to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/B/a55c23f912144f2c97cec3b2f8acf963 2024-11-22T03:42:22,577 DEBUG [StoreCloser-TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/B/ac98440d556545e0ae9f8aa814757c9d to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/B/ac98440d556545e0ae9f8aa814757c9d 2024-11-22T03:42:22,578 DEBUG [StoreCloser-TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/B/829066b589504d938b2fb6ac37a879b1 to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/B/829066b589504d938b2fb6ac37a879b1 2024-11-22T03:42:22,578 DEBUG [StoreCloser-TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/B/9390ed4698b340348b6f6a8156f9b8da to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/B/9390ed4698b340348b6f6a8156f9b8da 2024-11-22T03:42:22,579 DEBUG [StoreCloser-TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/B/26dc1f560f78442da4005f4162884cfe to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/B/26dc1f560f78442da4005f4162884cfe 2024-11-22T03:42:22,580 DEBUG [StoreCloser-TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/B/3bb2b7f488f54c5595ce617df05a1b7d to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/B/3bb2b7f488f54c5595ce617df05a1b7d 2024-11-22T03:42:22,580 DEBUG [StoreCloser-TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/B/554b916013a348b2b1739e3d0237de5f to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/B/554b916013a348b2b1739e3d0237de5f 2024-11-22T03:42:22,581 DEBUG [StoreCloser-TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/B/031726fec90d4c01ae917ca441ec14d5 to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/B/031726fec90d4c01ae917ca441ec14d5 2024-11-22T03:42:22,582 DEBUG [StoreCloser-TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/B/6b664097f71749ed85806a4b6afcb230 to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/B/6b664097f71749ed85806a4b6afcb230 2024-11-22T03:42:22,583 DEBUG [StoreCloser-TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/B/96ff445498f445dc8dcd512fb3da5c27 to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/B/96ff445498f445dc8dcd512fb3da5c27 2024-11-22T03:42:22,583 DEBUG [StoreCloser-TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/B/07d6e42780ed433d9e8b80bb72c2d006 to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/B/07d6e42780ed433d9e8b80bb72c2d006 2024-11-22T03:42:22,584 DEBUG [StoreCloser-TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/B/9bcaa526e82641cab5ce90d52f648996 to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/B/9bcaa526e82641cab5ce90d52f648996 2024-11-22T03:42:22,585 DEBUG [StoreCloser-TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/B/c9723ccbbfe24071a0dc29c27912627d to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/B/c9723ccbbfe24071a0dc29c27912627d 2024-11-22T03:42:22,586 DEBUG [StoreCloser-TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/B/9b17341d9a7c4c808ca45da794e72e12 to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/B/9b17341d9a7c4c808ca45da794e72e12 2024-11-22T03:42:22,586 DEBUG [StoreCloser-TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/B/985850e0381e47ad878559d3c83cf4aa to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/B/985850e0381e47ad878559d3c83cf4aa 2024-11-22T03:42:22,587 DEBUG [StoreCloser-TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/B/9f40ad9c40204d738e211c2f0eacf91e to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/B/9f40ad9c40204d738e211c2f0eacf91e 2024-11-22T03:42:22,588 DEBUG [StoreCloser-TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/B/383df294c68848c6a474c4249586c1a2 to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/B/383df294c68848c6a474c4249586c1a2 2024-11-22T03:42:22,588 DEBUG [StoreCloser-TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/B/34c1617ac80442279a56cf8de53a92a8 to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/B/34c1617ac80442279a56cf8de53a92a8 2024-11-22T03:42:22,589 DEBUG [StoreCloser-TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/B/2cecc979d6e748f497e69eb98b8e5e90 to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/B/2cecc979d6e748f497e69eb98b8e5e90 2024-11-22T03:42:22,590 DEBUG [StoreCloser-TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/B/889f23f7f1f945c9acbcae921888c97a to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/B/889f23f7f1f945c9acbcae921888c97a 2024-11-22T03:42:22,591 DEBUG [StoreCloser-TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/B/47cd5a9870994f05adbbd14fef32693d to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/B/47cd5a9870994f05adbbd14fef32693d 2024-11-22T03:42:22,592 DEBUG [StoreCloser-TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/C/6a0e4aa494e8439ba1cdc81dc334cf21, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/C/4e4851c784944e6482dec1e32d0e2ee3, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/C/59061dfed12c47f8a8d84488fa51ddc9, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/C/3d1667e5fc9749ccbf1b0b878518eb09, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/C/92a056a0d4ed4bceb6f96e9dc2bc8132, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/C/e897aad97196496696b4d82b2f2b5e80, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/C/864ce8fefe644bee8aa2e7d71cec23d2, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/C/e79f25f1f6a04dde8e1c6fe870de6db4, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/C/40a6b580d41747a581790e5e89fa6652, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/C/79b8b9a3799c41b99c8bf3d1b12f38fa, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/C/286c073c0a3d466e86b38934780fbd46, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/C/bc04f226748a48668bb72884ac2ba244, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/C/9cfaacff5cd64b3594653beaf1dca79e, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/C/a7152235b6094b9c8abcb0cf1c779870, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/C/956367ad9f6d4534b9da56e248803cee, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/C/3145934cf1ca4770afed9477b8de31de, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/C/6e42e52f3dcb4a38b274a0e52fe6c7eb, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/C/39c603a41ce04cfab5ed3a164df485ab, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/C/603ff7b096b547538d25c14e5ea6f147, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/C/311ebb17261249979ffa70985e116e07, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/C/1a0a6472d57b4d0da4bea93e692ca07c, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/C/8e4f9fea28654893a3f614d42c77d472, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/C/392b18f8879b47e3a54467440a77e8bb, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/C/150142ae2e7942f8bb86fbab1f5f3696, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/C/1da3fb94b44f4618b7b755006f46f028] to archive 2024-11-22T03:42:22,592 DEBUG [StoreCloser-TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-22T03:42:22,593 DEBUG [StoreCloser-TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/C/6a0e4aa494e8439ba1cdc81dc334cf21 to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/C/6a0e4aa494e8439ba1cdc81dc334cf21 2024-11-22T03:42:22,594 DEBUG [StoreCloser-TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/C/4e4851c784944e6482dec1e32d0e2ee3 to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/C/4e4851c784944e6482dec1e32d0e2ee3 2024-11-22T03:42:22,595 DEBUG [StoreCloser-TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/C/59061dfed12c47f8a8d84488fa51ddc9 to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/C/59061dfed12c47f8a8d84488fa51ddc9 2024-11-22T03:42:22,595 DEBUG [StoreCloser-TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/C/3d1667e5fc9749ccbf1b0b878518eb09 to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/C/3d1667e5fc9749ccbf1b0b878518eb09 2024-11-22T03:42:22,596 DEBUG [StoreCloser-TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/C/92a056a0d4ed4bceb6f96e9dc2bc8132 to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/C/92a056a0d4ed4bceb6f96e9dc2bc8132 2024-11-22T03:42:22,596 DEBUG [StoreCloser-TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/C/e897aad97196496696b4d82b2f2b5e80 to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/C/e897aad97196496696b4d82b2f2b5e80 2024-11-22T03:42:22,597 DEBUG [StoreCloser-TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/C/864ce8fefe644bee8aa2e7d71cec23d2 to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/C/864ce8fefe644bee8aa2e7d71cec23d2 2024-11-22T03:42:22,598 DEBUG [StoreCloser-TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/C/e79f25f1f6a04dde8e1c6fe870de6db4 to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/C/e79f25f1f6a04dde8e1c6fe870de6db4 2024-11-22T03:42:22,598 DEBUG [StoreCloser-TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/C/40a6b580d41747a581790e5e89fa6652 to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/C/40a6b580d41747a581790e5e89fa6652 2024-11-22T03:42:22,599 DEBUG [StoreCloser-TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/C/79b8b9a3799c41b99c8bf3d1b12f38fa to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/C/79b8b9a3799c41b99c8bf3d1b12f38fa 2024-11-22T03:42:22,600 DEBUG [StoreCloser-TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/C/286c073c0a3d466e86b38934780fbd46 to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/C/286c073c0a3d466e86b38934780fbd46 2024-11-22T03:42:22,600 DEBUG [StoreCloser-TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/C/bc04f226748a48668bb72884ac2ba244 to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/C/bc04f226748a48668bb72884ac2ba244 2024-11-22T03:42:22,601 DEBUG [StoreCloser-TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/C/9cfaacff5cd64b3594653beaf1dca79e to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/C/9cfaacff5cd64b3594653beaf1dca79e 2024-11-22T03:42:22,602 DEBUG [StoreCloser-TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/C/a7152235b6094b9c8abcb0cf1c779870 to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/C/a7152235b6094b9c8abcb0cf1c779870 2024-11-22T03:42:22,602 DEBUG [StoreCloser-TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/C/956367ad9f6d4534b9da56e248803cee to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/C/956367ad9f6d4534b9da56e248803cee 2024-11-22T03:42:22,603 DEBUG [StoreCloser-TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/C/3145934cf1ca4770afed9477b8de31de to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/C/3145934cf1ca4770afed9477b8de31de 2024-11-22T03:42:22,603 DEBUG [StoreCloser-TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/C/6e42e52f3dcb4a38b274a0e52fe6c7eb to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/C/6e42e52f3dcb4a38b274a0e52fe6c7eb 2024-11-22T03:42:22,604 DEBUG [StoreCloser-TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/C/39c603a41ce04cfab5ed3a164df485ab to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/C/39c603a41ce04cfab5ed3a164df485ab 2024-11-22T03:42:22,605 DEBUG [StoreCloser-TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/C/603ff7b096b547538d25c14e5ea6f147 to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/C/603ff7b096b547538d25c14e5ea6f147 2024-11-22T03:42:22,605 DEBUG [StoreCloser-TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/C/311ebb17261249979ffa70985e116e07 to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/C/311ebb17261249979ffa70985e116e07 2024-11-22T03:42:22,606 DEBUG [StoreCloser-TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/C/1a0a6472d57b4d0da4bea93e692ca07c to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/C/1a0a6472d57b4d0da4bea93e692ca07c 2024-11-22T03:42:22,607 DEBUG [StoreCloser-TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/C/8e4f9fea28654893a3f614d42c77d472 to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/C/8e4f9fea28654893a3f614d42c77d472 2024-11-22T03:42:22,607 DEBUG [StoreCloser-TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/C/392b18f8879b47e3a54467440a77e8bb to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/C/392b18f8879b47e3a54467440a77e8bb 2024-11-22T03:42:22,608 DEBUG [StoreCloser-TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/C/150142ae2e7942f8bb86fbab1f5f3696 to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/C/150142ae2e7942f8bb86fbab1f5f3696 2024-11-22T03:42:22,608 DEBUG [StoreCloser-TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/C/1da3fb94b44f4618b7b755006f46f028 to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/C/1da3fb94b44f4618b7b755006f46f028 2024-11-22T03:42:22,612 DEBUG [RS_CLOSE_REGION-regionserver/c85114ed5096:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/recovered.edits/397.seqid, newMaxSeqId=397, maxSeqId=1 2024-11-22T03:42:22,612 INFO [RS_CLOSE_REGION-regionserver/c85114ed5096:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795. 2024-11-22T03:42:22,612 DEBUG [RS_CLOSE_REGION-regionserver/c85114ed5096:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegion(1676): Region close journal for 4a7ce0cfb17f4dcc3dd24b2156963795: Waiting for close lock at 1732246941291Running coprocessor pre-close hooks at 1732246941291Disabling compacts and flushes for region at 1732246941291Disabling writes for close at 1732246941292 (+1 ms)Obtaining lock to block concurrent updates at 1732246941292Preparing flush snapshotting stores in 4a7ce0cfb17f4dcc3dd24b2156963795 at 1732246941292Finished memstore snapshotting TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795., syncing WAL and waiting on mvcc, flushsize=dataSize=20610, getHeapSize=54720, getOffHeapSize=0, getCellsCount=450 at 1732246941292Flushing stores of TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795. at 1732246941293 (+1 ms)Flushing 4a7ce0cfb17f4dcc3dd24b2156963795/A: creating writer at 1732246941293Flushing 4a7ce0cfb17f4dcc3dd24b2156963795/A: appending metadata at 1732246941295 (+2 ms)Flushing 4a7ce0cfb17f4dcc3dd24b2156963795/A: closing flushed file at 1732246941295Flushing 4a7ce0cfb17f4dcc3dd24b2156963795/B: creating writer at 1732246941709 (+414 ms)Flushing 4a7ce0cfb17f4dcc3dd24b2156963795/B: appending metadata at 1732246941713 (+4 ms)Flushing 4a7ce0cfb17f4dcc3dd24b2156963795/B: closing flushed file at 1732246941713Flushing 4a7ce0cfb17f4dcc3dd24b2156963795/C: creating writer at 1732246942128 (+415 ms)Flushing 4a7ce0cfb17f4dcc3dd24b2156963795/C: appending metadata at 1732246942131 (+3 ms)Flushing 4a7ce0cfb17f4dcc3dd24b2156963795/C: closing flushed file at 1732246942131Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2531ba0b: reopening flushed file at 1732246942539 (+408 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5667a1df: reopening flushed file at 1732246942543 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3b1a710f: reopening flushed file at 1732246942546 (+3 ms)Finished flush of dataSize ~20.13 KB/20610, heapSize ~53.44 KB/54720, currentSize=0 B/0 for 4a7ce0cfb17f4dcc3dd24b2156963795 in 1258ms, sequenceid=394, compaction requested=true at 1732246942550 (+4 ms)Writing region close event to WAL at 1732246942609 (+59 ms)Running coprocessor post-close hooks at 1732246942612 (+3 ms)Closed at 1732246942612 2024-11-22T03:42:22,614 INFO [RS_CLOSE_REGION-regionserver/c85114ed5096:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] handler.UnassignRegionHandler(157): Closed 4a7ce0cfb17f4dcc3dd24b2156963795 2024-11-22T03:42:22,614 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=139 updating hbase:meta row=4a7ce0cfb17f4dcc3dd24b2156963795, regionState=CLOSED 2024-11-22T03:42:22,615 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=140, ppid=139, state=RUNNABLE, hasLock=false; CloseRegionProcedure 4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 because future has completed 2024-11-22T03:42:22,617 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=140, resume processing ppid=139 2024-11-22T03:42:22,617 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=140, ppid=139, state=SUCCESS, hasLock=false; CloseRegionProcedure 4a7ce0cfb17f4dcc3dd24b2156963795, server=c85114ed5096,42545,1732246786482 in 1.4770 sec 2024-11-22T03:42:22,619 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=139, resume processing ppid=138 2024-11-22T03:42:22,619 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=139, ppid=138, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=4a7ce0cfb17f4dcc3dd24b2156963795, UNASSIGN in 1.4800 sec 2024-11-22T03:42:22,621 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=138, resume processing ppid=137 2024-11-22T03:42:22,621 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=138, ppid=137, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.4840 sec 2024-11-22T03:42:22,622 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732246942622"}]},"ts":"1732246942622"} 2024-11-22T03:42:22,623 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-22T03:42:22,623 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-22T03:42:22,624 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=137, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.4920 sec 2024-11-22T03:42:23,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-22T03:42:23,279 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-22T03:42:23,279 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42627 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-22T03:42:23,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42627 {}] procedure2.ProcedureExecutor(1139): Stored pid=141, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-22T03:42:23,281 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=141, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-22T03:42:23,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-22T03:42:23,282 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=141, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-22T03:42:23,283 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795 2024-11-22T03:42:23,285 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/A, FileablePath, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/B, FileablePath, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/C, FileablePath, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/recovered.edits] 2024-11-22T03:42:23,287 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/A/34ab25aeaa6744ac87ee19623d03ec17 to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/A/34ab25aeaa6744ac87ee19623d03ec17 2024-11-22T03:42:23,288 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/A/4669d86ae53c49d4b3b4cf36483e9448 to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/A/4669d86ae53c49d4b3b4cf36483e9448 2024-11-22T03:42:23,288 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/A/932040e0a0f84b3da137ea5ac99f87b9 to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/A/932040e0a0f84b3da137ea5ac99f87b9 2024-11-22T03:42:23,290 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/B/3e5ecaa50d4b45ffb3d027dc6016b2fd to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/B/3e5ecaa50d4b45ffb3d027dc6016b2fd 2024-11-22T03:42:23,291 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/B/6210bd8e2a164e3ba3e20ac88744c464 to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/B/6210bd8e2a164e3ba3e20ac88744c464 2024-11-22T03:42:23,291 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/B/c8a2fdb78834451c844cb8cf65b14f52 to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/B/c8a2fdb78834451c844cb8cf65b14f52 2024-11-22T03:42:23,293 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/C/3a08480090f2453882beafbae5c06c0c to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/C/3a08480090f2453882beafbae5c06c0c 2024-11-22T03:42:23,294 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/C/a4a5d2874e2e4846a2f5d15a5968a691 to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/C/a4a5d2874e2e4846a2f5d15a5968a691 2024-11-22T03:42:23,294 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/C/acbb92838a044d72acf1d49e79a42cc9 to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/C/acbb92838a044d72acf1d49e79a42cc9 2024-11-22T03:42:23,296 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/recovered.edits/397.seqid to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795/recovered.edits/397.seqid 2024-11-22T03:42:23,296 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/4a7ce0cfb17f4dcc3dd24b2156963795 2024-11-22T03:42:23,296 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-22T03:42:23,298 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=141, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-22T03:42:23,300 WARN [PEWorker-1 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-22T03:42:23,301 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-22T03:42:23,302 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=141, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-22T03:42:23,302 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-22T03:42:23,302 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732246943302"}]},"ts":"9223372036854775807"} 2024-11-22T03:42:23,303 INFO [PEWorker-1 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-22T03:42:23,304 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 4a7ce0cfb17f4dcc3dd24b2156963795, NAME => 'TestAcidGuarantees,,1732246908584.4a7ce0cfb17f4dcc3dd24b2156963795.', STARTKEY => '', ENDKEY => ''}] 2024-11-22T03:42:23,304 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-22T03:42:23,304 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732246943304"}]},"ts":"9223372036854775807"} 2024-11-22T03:42:23,305 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-22T03:42:23,306 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(133): Finished pid=141, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-22T03:42:23,307 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=141, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 27 msec 2024-11-22T03:42:23,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-22T03:42:23,389 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-22T03:42:23,389 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-22T03:42:23,396 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testScanAtomicity Thread=238 (was 241), OpenFileDescriptor=443 (was 461), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=365 (was 381), ProcessCount=11 (was 11), AvailableMemoryMB=3053 (was 3232) 2024-11-22T03:42:23,403 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testMobGetAtomicity Thread=238, OpenFileDescriptor=443, MaxFileDescriptor=1048576, SystemLoadAverage=365, ProcessCount=11, AvailableMemoryMB=3053 2024-11-22T03:42:23,404 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42627 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-22T03:42:23,404 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42627 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-22T03:42:23,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42627 {}] procedure2.ProcedureExecutor(1139): Stored pid=142, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-22T03:42:23,406 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=142, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-22T03:42:23,406 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:42:23,406 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42627 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 142 2024-11-22T03:42:23,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-11-22T03:42:23,407 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=142, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-22T03:42:23,411 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742304_1480 (size=959) 2024-11-22T03:42:23,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-11-22T03:42:23,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-11-22T03:42:23,813 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 8422287be02eb151a34ae7fe752afee0, NAME => 'TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a 2024-11-22T03:42:23,817 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742305_1481 (size=53) 2024-11-22T03:42:24,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-11-22T03:42:24,220 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-22T03:42:24,220 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 8422287be02eb151a34ae7fe752afee0, disabling compactions & flushes 2024-11-22T03:42:24,220 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0. 2024-11-22T03:42:24,220 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0. 2024-11-22T03:42:24,220 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0. after waiting 0 ms 2024-11-22T03:42:24,220 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0. 2024-11-22T03:42:24,220 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0. 2024-11-22T03:42:24,220 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 8422287be02eb151a34ae7fe752afee0: Waiting for close lock at 1732246944220Disabling compacts and flushes for region at 1732246944220Disabling writes for close at 1732246944220Writing region close event to WAL at 1732246944220Closed at 1732246944220 2024-11-22T03:42:24,223 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=142, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-22T03:42:24,224 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732246944223"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732246944223"}]},"ts":"1732246944223"} 2024-11-22T03:42:24,228 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-22T03:42:24,230 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=142, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-22T03:42:24,230 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732246944230"}]},"ts":"1732246944230"} 2024-11-22T03:42:24,232 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-22T03:42:24,232 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=143, ppid=142, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=8422287be02eb151a34ae7fe752afee0, ASSIGN}] 2024-11-22T03:42:24,233 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=143, ppid=142, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=8422287be02eb151a34ae7fe752afee0, ASSIGN 2024-11-22T03:42:24,233 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=143, ppid=142, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=8422287be02eb151a34ae7fe752afee0, ASSIGN; state=OFFLINE, location=c85114ed5096,42545,1732246786482; forceNewPlan=false, retain=false 2024-11-22T03:42:24,385 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=143 updating hbase:meta row=8422287be02eb151a34ae7fe752afee0, regionState=OPENING, regionLocation=c85114ed5096,42545,1732246786482 2024-11-22T03:42:24,388 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=143, ppid=142, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=8422287be02eb151a34ae7fe752afee0, ASSIGN because future has completed 2024-11-22T03:42:24,390 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=144, ppid=143, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482}] 2024-11-22T03:42:24,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-11-22T03:42:24,550 INFO [RS_OPEN_REGION-regionserver/c85114ed5096:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0. 2024-11-22T03:42:24,550 DEBUG [RS_OPEN_REGION-regionserver/c85114ed5096:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(7752): Opening region: {ENCODED => 8422287be02eb151a34ae7fe752afee0, NAME => 'TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0.', STARTKEY => '', ENDKEY => ''} 2024-11-22T03:42:24,551 DEBUG [RS_OPEN_REGION-regionserver/c85114ed5096:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 8422287be02eb151a34ae7fe752afee0 2024-11-22T03:42:24,551 DEBUG [RS_OPEN_REGION-regionserver/c85114ed5096:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-22T03:42:24,552 DEBUG [RS_OPEN_REGION-regionserver/c85114ed5096:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(7794): checking encryption for 8422287be02eb151a34ae7fe752afee0 2024-11-22T03:42:24,552 DEBUG [RS_OPEN_REGION-regionserver/c85114ed5096:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(7797): checking classloading for 8422287be02eb151a34ae7fe752afee0 2024-11-22T03:42:24,554 INFO [StoreOpener-8422287be02eb151a34ae7fe752afee0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 8422287be02eb151a34ae7fe752afee0 2024-11-22T03:42:24,556 INFO [StoreOpener-8422287be02eb151a34ae7fe752afee0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8422287be02eb151a34ae7fe752afee0 columnFamilyName A 2024-11-22T03:42:24,556 DEBUG [StoreOpener-8422287be02eb151a34ae7fe752afee0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:42:24,557 INFO [StoreOpener-8422287be02eb151a34ae7fe752afee0-1 {}] regionserver.HStore(327): Store=8422287be02eb151a34ae7fe752afee0/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-22T03:42:24,557 INFO [StoreOpener-8422287be02eb151a34ae7fe752afee0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 8422287be02eb151a34ae7fe752afee0 2024-11-22T03:42:24,558 INFO [StoreOpener-8422287be02eb151a34ae7fe752afee0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8422287be02eb151a34ae7fe752afee0 columnFamilyName B 2024-11-22T03:42:24,558 DEBUG [StoreOpener-8422287be02eb151a34ae7fe752afee0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:42:24,558 INFO [StoreOpener-8422287be02eb151a34ae7fe752afee0-1 {}] regionserver.HStore(327): Store=8422287be02eb151a34ae7fe752afee0/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-22T03:42:24,558 INFO [StoreOpener-8422287be02eb151a34ae7fe752afee0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 8422287be02eb151a34ae7fe752afee0 2024-11-22T03:42:24,559 INFO [StoreOpener-8422287be02eb151a34ae7fe752afee0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8422287be02eb151a34ae7fe752afee0 columnFamilyName C 2024-11-22T03:42:24,559 DEBUG [StoreOpener-8422287be02eb151a34ae7fe752afee0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:42:24,559 INFO [StoreOpener-8422287be02eb151a34ae7fe752afee0-1 {}] regionserver.HStore(327): Store=8422287be02eb151a34ae7fe752afee0/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-22T03:42:24,560 DEBUG [RS_OPEN_REGION-regionserver/c85114ed5096:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(1038): replaying wal for 8422287be02eb151a34ae7fe752afee0 2024-11-22T03:42:24,560 DEBUG [RS_OPEN_REGION-regionserver/c85114ed5096:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0 2024-11-22T03:42:24,560 DEBUG [RS_OPEN_REGION-regionserver/c85114ed5096:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0 2024-11-22T03:42:24,561 DEBUG [RS_OPEN_REGION-regionserver/c85114ed5096:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(1048): stopping wal replay for 8422287be02eb151a34ae7fe752afee0 2024-11-22T03:42:24,561 DEBUG [RS_OPEN_REGION-regionserver/c85114ed5096:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(1060): Cleaning up temporary data for 8422287be02eb151a34ae7fe752afee0 2024-11-22T03:42:24,562 DEBUG [RS_OPEN_REGION-regionserver/c85114ed5096:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-22T03:42:24,562 DEBUG [RS_OPEN_REGION-regionserver/c85114ed5096:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(1093): writing seq id for 8422287be02eb151a34ae7fe752afee0 2024-11-22T03:42:24,564 DEBUG [RS_OPEN_REGION-regionserver/c85114ed5096:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-22T03:42:24,564 INFO [RS_OPEN_REGION-regionserver/c85114ed5096:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(1114): Opened 8422287be02eb151a34ae7fe752afee0; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74558151, jitterRate=0.11100302636623383}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-22T03:42:24,564 DEBUG [RS_OPEN_REGION-regionserver/c85114ed5096:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 8422287be02eb151a34ae7fe752afee0 2024-11-22T03:42:24,564 DEBUG [RS_OPEN_REGION-regionserver/c85114ed5096:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(1006): Region open journal for 8422287be02eb151a34ae7fe752afee0: Running coprocessor pre-open hook at 1732246944552Writing region info on filesystem at 1732246944552Initializing all the Stores at 1732246944553 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732246944553Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732246944553Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732246944554 (+1 ms)Cleaning up temporary data from old regions at 1732246944561 (+7 ms)Running coprocessor post-open hooks at 1732246944564 (+3 ms)Region opened successfully at 1732246944564 2024-11-22T03:42:24,565 INFO [RS_OPEN_REGION-regionserver/c85114ed5096:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., pid=144, masterSystemTime=1732246944543 2024-11-22T03:42:24,566 DEBUG [RS_OPEN_REGION-regionserver/c85114ed5096:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0. 2024-11-22T03:42:24,566 INFO [RS_OPEN_REGION-regionserver/c85114ed5096:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0. 2024-11-22T03:42:24,567 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=143 updating hbase:meta row=8422287be02eb151a34ae7fe752afee0, regionState=OPEN, openSeqNum=2, regionLocation=c85114ed5096,42545,1732246786482 2024-11-22T03:42:24,568 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=144, ppid=143, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 because future has completed 2024-11-22T03:42:24,570 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=144, resume processing ppid=143 2024-11-22T03:42:24,570 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=144, ppid=143, state=SUCCESS, hasLock=false; OpenRegionProcedure 8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 in 179 msec 2024-11-22T03:42:24,571 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=143, resume processing ppid=142 2024-11-22T03:42:24,571 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=143, ppid=142, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=8422287be02eb151a34ae7fe752afee0, ASSIGN in 338 msec 2024-11-22T03:42:24,571 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=142, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-22T03:42:24,572 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732246944571"}]},"ts":"1732246944571"} 2024-11-22T03:42:24,572 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-22T03:42:24,573 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=142, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-22T03:42:24,574 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=142, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1690 sec 2024-11-22T03:42:25,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-11-22T03:42:25,549 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-22T03:42:25,550 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@348042de, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T03:42:25,550 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c85114ed5096,42627,-1 for getting cluster id 2024-11-22T03:42:25,550 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-22T03:42:25,551 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '67035b7f-c649-44a8-9db7-17adff49cedb' 2024-11-22T03:42:25,551 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-22T03:42:25,551 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "67035b7f-c649-44a8-9db7-17adff49cedb" 2024-11-22T03:42:25,551 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2e7186a8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T03:42:25,551 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c85114ed5096,42627,-1] 2024-11-22T03:42:25,551 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-22T03:42:25,551 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T03:42:25,552 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43666, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-22T03:42:25,552 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@56b639d0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T03:42:25,553 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-22T03:42:25,553 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c85114ed5096,42545,1732246786482, seqNum=-1] 2024-11-22T03:42:25,553 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-22T03:42:25,554 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37344, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-22T03:42:25,554 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-22T03:42:25,555 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncConnectionImpl(321): The fetched master address is c85114ed5096,42627,1732246785796 2024-11-22T03:42:25,555 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@a6b1e9c 2024-11-22T03:42:25,555 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-22T03:42:25,556 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43678, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-22T03:42:25,557 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42627 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-22T03:42:25,557 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42627 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-22T03:42:25,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42627 {}] procedure2.ProcedureExecutor(1139): Stored pid=145, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-22T03:42:25,566 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742306_1482 (size=995) 2024-11-22T03:42:25,967 DEBUG [PEWorker-4 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.959 2024-11-22T03:42:25,967 INFO [PEWorker-4 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.995 2024-11-22T03:42:25,969 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=146, ppid=145, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-22T03:42:25,971 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=147, ppid=146, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=8422287be02eb151a34ae7fe752afee0, REOPEN/MOVE}] 2024-11-22T03:42:25,972 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=147, ppid=146, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=8422287be02eb151a34ae7fe752afee0, REOPEN/MOVE 2024-11-22T03:42:25,973 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=147 updating hbase:meta row=8422287be02eb151a34ae7fe752afee0, regionState=CLOSING, regionLocation=c85114ed5096,42545,1732246786482 2024-11-22T03:42:25,974 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=147, ppid=146, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=8422287be02eb151a34ae7fe752afee0, REOPEN/MOVE because future has completed 2024-11-22T03:42:25,974 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-22T03:42:25,974 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=148, ppid=147, state=RUNNABLE, hasLock=false; CloseRegionProcedure 8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482}] 2024-11-22T03:42:26,127 INFO [RS_CLOSE_REGION-regionserver/c85114ed5096:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] handler.UnassignRegionHandler(122): Close 8422287be02eb151a34ae7fe752afee0 2024-11-22T03:42:26,127 DEBUG [RS_CLOSE_REGION-regionserver/c85114ed5096:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-22T03:42:26,127 DEBUG [RS_CLOSE_REGION-regionserver/c85114ed5096:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.HRegion(1722): Closing 8422287be02eb151a34ae7fe752afee0, disabling compactions & flushes 2024-11-22T03:42:26,127 INFO [RS_CLOSE_REGION-regionserver/c85114ed5096:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0. 2024-11-22T03:42:26,127 DEBUG [RS_CLOSE_REGION-regionserver/c85114ed5096:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0. 2024-11-22T03:42:26,127 DEBUG [RS_CLOSE_REGION-regionserver/c85114ed5096:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0. after waiting 0 ms 2024-11-22T03:42:26,128 DEBUG [RS_CLOSE_REGION-regionserver/c85114ed5096:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0. 2024-11-22T03:42:26,135 DEBUG [RS_CLOSE_REGION-regionserver/c85114ed5096:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-22T03:42:26,136 INFO [RS_CLOSE_REGION-regionserver/c85114ed5096:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0. 2024-11-22T03:42:26,136 DEBUG [RS_CLOSE_REGION-regionserver/c85114ed5096:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.HRegion(1676): Region close journal for 8422287be02eb151a34ae7fe752afee0: Waiting for close lock at 1732246946127Running coprocessor pre-close hooks at 1732246946127Disabling compacts and flushes for region at 1732246946127Disabling writes for close at 1732246946128 (+1 ms)Writing region close event to WAL at 1732246946128Running coprocessor post-close hooks at 1732246946136 (+8 ms)Closed at 1732246946136 2024-11-22T03:42:26,136 WARN [RS_CLOSE_REGION-regionserver/c85114ed5096:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.HRegionServer(3299): Not adding moved region record: 8422287be02eb151a34ae7fe752afee0 to self. 2024-11-22T03:42:26,138 INFO [RS_CLOSE_REGION-regionserver/c85114ed5096:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] handler.UnassignRegionHandler(157): Closed 8422287be02eb151a34ae7fe752afee0 2024-11-22T03:42:26,139 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=147 updating hbase:meta row=8422287be02eb151a34ae7fe752afee0, regionState=CLOSED 2024-11-22T03:42:26,141 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=148, ppid=147, state=RUNNABLE, hasLock=false; CloseRegionProcedure 8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 because future has completed 2024-11-22T03:42:26,143 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=148, resume processing ppid=147 2024-11-22T03:42:26,144 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=148, ppid=147, state=SUCCESS, hasLock=false; CloseRegionProcedure 8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 in 167 msec 2024-11-22T03:42:26,144 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=147, ppid=146, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=8422287be02eb151a34ae7fe752afee0, REOPEN/MOVE; state=CLOSED, location=c85114ed5096,42545,1732246786482; forceNewPlan=false, retain=true 2024-11-22T03:42:26,188 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-22T03:42:26,295 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=147 updating hbase:meta row=8422287be02eb151a34ae7fe752afee0, regionState=OPENING, regionLocation=c85114ed5096,42545,1732246786482 2024-11-22T03:42:26,299 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=147, ppid=146, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=8422287be02eb151a34ae7fe752afee0, REOPEN/MOVE because future has completed 2024-11-22T03:42:26,300 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=149, ppid=147, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482}] 2024-11-22T03:42:26,460 INFO [RS_OPEN_REGION-regionserver/c85114ed5096:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0. 2024-11-22T03:42:26,460 DEBUG [RS_OPEN_REGION-regionserver/c85114ed5096:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(7752): Opening region: {ENCODED => 8422287be02eb151a34ae7fe752afee0, NAME => 'TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0.', STARTKEY => '', ENDKEY => ''} 2024-11-22T03:42:26,460 DEBUG [RS_OPEN_REGION-regionserver/c85114ed5096:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 8422287be02eb151a34ae7fe752afee0 2024-11-22T03:42:26,461 DEBUG [RS_OPEN_REGION-regionserver/c85114ed5096:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-22T03:42:26,461 DEBUG [RS_OPEN_REGION-regionserver/c85114ed5096:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(7794): checking encryption for 8422287be02eb151a34ae7fe752afee0 2024-11-22T03:42:26,461 DEBUG [RS_OPEN_REGION-regionserver/c85114ed5096:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(7797): checking classloading for 8422287be02eb151a34ae7fe752afee0 2024-11-22T03:42:26,463 INFO [StoreOpener-8422287be02eb151a34ae7fe752afee0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 8422287be02eb151a34ae7fe752afee0 2024-11-22T03:42:26,464 INFO [StoreOpener-8422287be02eb151a34ae7fe752afee0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8422287be02eb151a34ae7fe752afee0 columnFamilyName A 2024-11-22T03:42:26,466 DEBUG [StoreOpener-8422287be02eb151a34ae7fe752afee0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:42:26,466 INFO [StoreOpener-8422287be02eb151a34ae7fe752afee0-1 {}] regionserver.HStore(327): Store=8422287be02eb151a34ae7fe752afee0/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-22T03:42:26,466 INFO [StoreOpener-8422287be02eb151a34ae7fe752afee0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 8422287be02eb151a34ae7fe752afee0 2024-11-22T03:42:26,467 INFO [StoreOpener-8422287be02eb151a34ae7fe752afee0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8422287be02eb151a34ae7fe752afee0 columnFamilyName B 2024-11-22T03:42:26,467 DEBUG [StoreOpener-8422287be02eb151a34ae7fe752afee0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:42:26,468 INFO [StoreOpener-8422287be02eb151a34ae7fe752afee0-1 {}] regionserver.HStore(327): Store=8422287be02eb151a34ae7fe752afee0/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-22T03:42:26,468 INFO [StoreOpener-8422287be02eb151a34ae7fe752afee0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 8422287be02eb151a34ae7fe752afee0 2024-11-22T03:42:26,468 INFO [StoreOpener-8422287be02eb151a34ae7fe752afee0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8422287be02eb151a34ae7fe752afee0 columnFamilyName C 2024-11-22T03:42:26,468 DEBUG [StoreOpener-8422287be02eb151a34ae7fe752afee0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:42:26,469 INFO [StoreOpener-8422287be02eb151a34ae7fe752afee0-1 {}] regionserver.HStore(327): Store=8422287be02eb151a34ae7fe752afee0/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-22T03:42:26,469 DEBUG [RS_OPEN_REGION-regionserver/c85114ed5096:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(1038): replaying wal for 8422287be02eb151a34ae7fe752afee0 2024-11-22T03:42:26,469 DEBUG [RS_OPEN_REGION-regionserver/c85114ed5096:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0 2024-11-22T03:42:26,470 DEBUG [RS_OPEN_REGION-regionserver/c85114ed5096:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0 2024-11-22T03:42:26,471 DEBUG [RS_OPEN_REGION-regionserver/c85114ed5096:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(1048): stopping wal replay for 8422287be02eb151a34ae7fe752afee0 2024-11-22T03:42:26,471 DEBUG [RS_OPEN_REGION-regionserver/c85114ed5096:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(1060): Cleaning up temporary data for 8422287be02eb151a34ae7fe752afee0 2024-11-22T03:42:26,472 DEBUG [RS_OPEN_REGION-regionserver/c85114ed5096:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-22T03:42:26,473 DEBUG [RS_OPEN_REGION-regionserver/c85114ed5096:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(1093): writing seq id for 8422287be02eb151a34ae7fe752afee0 2024-11-22T03:42:26,474 INFO [RS_OPEN_REGION-regionserver/c85114ed5096:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(1114): Opened 8422287be02eb151a34ae7fe752afee0; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72737851, jitterRate=0.08387844264507294}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-22T03:42:26,474 DEBUG [RS_OPEN_REGION-regionserver/c85114ed5096:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 8422287be02eb151a34ae7fe752afee0 2024-11-22T03:42:26,474 DEBUG [RS_OPEN_REGION-regionserver/c85114ed5096:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(1006): Region open journal for 8422287be02eb151a34ae7fe752afee0: Running coprocessor pre-open hook at 1732246946461Writing region info on filesystem at 1732246946461Initializing all the Stores at 1732246946462 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732246946462Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732246946463 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732246946463Cleaning up temporary data from old regions at 1732246946471 (+8 ms)Running coprocessor post-open hooks at 1732246946474 (+3 ms)Region opened successfully at 1732246946474 2024-11-22T03:42:26,475 INFO [RS_OPEN_REGION-regionserver/c85114ed5096:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., pid=149, masterSystemTime=1732246946455 2024-11-22T03:42:26,477 DEBUG [RS_OPEN_REGION-regionserver/c85114ed5096:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0. 2024-11-22T03:42:26,477 INFO [RS_OPEN_REGION-regionserver/c85114ed5096:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0. 2024-11-22T03:42:26,478 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=147 updating hbase:meta row=8422287be02eb151a34ae7fe752afee0, regionState=OPEN, openSeqNum=5, regionLocation=c85114ed5096,42545,1732246786482 2024-11-22T03:42:26,479 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=149, ppid=147, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 because future has completed 2024-11-22T03:42:26,483 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=149, resume processing ppid=147 2024-11-22T03:42:26,483 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=149, ppid=147, state=SUCCESS, hasLock=false; OpenRegionProcedure 8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 in 181 msec 2024-11-22T03:42:26,486 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=147, resume processing ppid=146 2024-11-22T03:42:26,486 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=147, ppid=146, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=8422287be02eb151a34ae7fe752afee0, REOPEN/MOVE in 512 msec 2024-11-22T03:42:26,489 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=146, resume processing ppid=145 2024-11-22T03:42:26,489 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=146, ppid=145, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 518 msec 2024-11-22T03:42:26,492 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=145, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 933 msec 2024-11-22T03:42:26,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=145 2024-11-22T03:42:26,493 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-22T03:42:26,494 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@24560e66, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T03:42:26,494 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c85114ed5096,42627,-1 for getting cluster id 2024-11-22T03:42:26,494 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-22T03:42:26,495 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '67035b7f-c649-44a8-9db7-17adff49cedb' 2024-11-22T03:42:26,495 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-22T03:42:26,495 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "67035b7f-c649-44a8-9db7-17adff49cedb" 2024-11-22T03:42:26,495 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@735c8f71, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T03:42:26,495 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c85114ed5096,42627,-1] 2024-11-22T03:42:26,495 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-22T03:42:26,495 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T03:42:26,496 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43696, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-22T03:42:26,496 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@39aa278, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T03:42:26,497 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7e83be8c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T03:42:26,497 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c85114ed5096,42627,-1 for getting cluster id 2024-11-22T03:42:26,497 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-22T03:42:26,497 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '67035b7f-c649-44a8-9db7-17adff49cedb' 2024-11-22T03:42:26,497 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-22T03:42:26,497 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "67035b7f-c649-44a8-9db7-17adff49cedb" 2024-11-22T03:42:26,497 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@62a83e6c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T03:42:26,497 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c85114ed5096,42627,-1] 2024-11-22T03:42:26,498 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-22T03:42:26,498 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T03:42:26,498 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43714, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-22T03:42:26,499 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7f6ec157, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T03:42:26,499 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@49c8d9ab, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T03:42:26,499 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c85114ed5096,42627,-1 for getting cluster id 2024-11-22T03:42:26,499 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-22T03:42:26,500 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '67035b7f-c649-44a8-9db7-17adff49cedb' 2024-11-22T03:42:26,500 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-22T03:42:26,500 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "67035b7f-c649-44a8-9db7-17adff49cedb" 2024-11-22T03:42:26,500 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@58799eab, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T03:42:26,500 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c85114ed5096,42627,-1] 2024-11-22T03:42:26,500 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-22T03:42:26,500 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T03:42:26,500 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43736, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-22T03:42:26,501 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4fbf155b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T03:42:26,501 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@37f0cd5b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T03:42:26,501 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c85114ed5096,42627,-1 for getting cluster id 2024-11-22T03:42:26,502 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-22T03:42:26,502 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '67035b7f-c649-44a8-9db7-17adff49cedb' 2024-11-22T03:42:26,502 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-22T03:42:26,502 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "67035b7f-c649-44a8-9db7-17adff49cedb" 2024-11-22T03:42:26,502 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32765772, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T03:42:26,502 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c85114ed5096,42627,-1] 2024-11-22T03:42:26,502 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-22T03:42:26,502 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T03:42:26,503 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43762, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-22T03:42:26,503 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@31b439b5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T03:42:26,504 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@41947491, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T03:42:26,504 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c85114ed5096,42627,-1 for getting cluster id 2024-11-22T03:42:26,504 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-22T03:42:26,504 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '67035b7f-c649-44a8-9db7-17adff49cedb' 2024-11-22T03:42:26,504 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-22T03:42:26,505 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "67035b7f-c649-44a8-9db7-17adff49cedb" 2024-11-22T03:42:26,505 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7dca29cb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T03:42:26,505 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c85114ed5096,42627,-1] 2024-11-22T03:42:26,505 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-22T03:42:26,505 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T03:42:26,505 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43780, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-22T03:42:26,506 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@329fd335, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T03:42:26,506 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@19bc8537, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T03:42:26,506 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c85114ed5096,42627,-1 for getting cluster id 2024-11-22T03:42:26,506 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-22T03:42:26,507 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '67035b7f-c649-44a8-9db7-17adff49cedb' 2024-11-22T03:42:26,507 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-22T03:42:26,507 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "67035b7f-c649-44a8-9db7-17adff49cedb" 2024-11-22T03:42:26,507 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2c4a422a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T03:42:26,507 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c85114ed5096,42627,-1] 2024-11-22T03:42:26,507 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-22T03:42:26,508 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T03:42:26,508 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43802, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-22T03:42:26,508 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6bf17757, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T03:42:26,509 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7595d14b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T03:42:26,509 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c85114ed5096,42627,-1 for getting cluster id 2024-11-22T03:42:26,509 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-22T03:42:26,509 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '67035b7f-c649-44a8-9db7-17adff49cedb' 2024-11-22T03:42:26,510 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-22T03:42:26,510 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "67035b7f-c649-44a8-9db7-17adff49cedb" 2024-11-22T03:42:26,510 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4d9d227e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T03:42:26,510 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c85114ed5096,42627,-1] 2024-11-22T03:42:26,510 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-22T03:42:26,510 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T03:42:26,511 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43822, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-22T03:42:26,511 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@330e89fe, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T03:42:26,511 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2bf79e45, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T03:42:26,511 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c85114ed5096,42627,-1 for getting cluster id 2024-11-22T03:42:26,512 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-22T03:42:26,512 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '67035b7f-c649-44a8-9db7-17adff49cedb' 2024-11-22T03:42:26,512 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-22T03:42:26,512 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "67035b7f-c649-44a8-9db7-17adff49cedb" 2024-11-22T03:42:26,512 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2f2da159, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T03:42:26,512 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c85114ed5096,42627,-1] 2024-11-22T03:42:26,512 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-22T03:42:26,513 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T03:42:26,513 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43844, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-22T03:42:26,513 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1582762c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T03:42:26,514 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@23d0e9c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T03:42:26,514 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c85114ed5096,42627,-1 for getting cluster id 2024-11-22T03:42:26,514 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-22T03:42:26,515 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '67035b7f-c649-44a8-9db7-17adff49cedb' 2024-11-22T03:42:26,515 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-22T03:42:26,515 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "67035b7f-c649-44a8-9db7-17adff49cedb" 2024-11-22T03:42:26,515 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7fde0f2e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T03:42:26,515 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c85114ed5096,42627,-1] 2024-11-22T03:42:26,515 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-22T03:42:26,515 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T03:42:26,516 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43860, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-22T03:42:26,516 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1c464b14, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T03:42:26,517 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@f9a2803, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T03:42:26,517 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c85114ed5096,42627,-1 for getting cluster id 2024-11-22T03:42:26,517 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-22T03:42:26,517 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '67035b7f-c649-44a8-9db7-17adff49cedb' 2024-11-22T03:42:26,517 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-22T03:42:26,517 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "67035b7f-c649-44a8-9db7-17adff49cedb" 2024-11-22T03:42:26,517 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@482f0277, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T03:42:26,517 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c85114ed5096,42627,-1] 2024-11-22T03:42:26,518 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-22T03:42:26,518 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T03:42:26,518 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43870, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-22T03:42:26,519 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1d102dea, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T03:42:26,519 DEBUG [Thread-2191 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-22T03:42:26,519 DEBUG [Thread-2197 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-22T03:42:26,520 DEBUG [Thread-2195 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-22T03:42:26,520 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c85114ed5096,42545,1732246786482, seqNum=-1] 2024-11-22T03:42:26,520 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-22T03:42:26,520 DEBUG [Thread-2188 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-22T03:42:26,520 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c85114ed5096,42545,1732246786482, seqNum=-1] 2024-11-22T03:42:26,520 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c85114ed5096,42545,1732246786482, seqNum=-1] 2024-11-22T03:42:26,520 DEBUG [Thread-2182 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-22T03:42:26,520 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-22T03:42:26,520 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-22T03:42:26,521 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c85114ed5096,42545,1732246786482, seqNum=-1] 2024-11-22T03:42:26,521 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-22T03:42:26,521 DEBUG [Thread-2180 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-22T03:42:26,521 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42627 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-22T03:42:26,521 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37352, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-22T03:42:26,521 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37366, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-22T03:42:26,521 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c85114ed5096,42545,1732246786482, seqNum=-1] 2024-11-22T03:42:26,522 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-22T03:42:26,522 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37358, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-22T03:42:26,522 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c85114ed5096,42545,1732246786482, seqNum=-1] 2024-11-22T03:42:26,522 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-22T03:42:26,522 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37382, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-22T03:42:26,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42627 {}] procedure2.ProcedureExecutor(1139): Stored pid=150, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees 2024-11-22T03:42:26,522 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37384, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-22T03:42:26,522 DEBUG [Thread-2199 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-22T03:42:26,523 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5] 2024-11-22T03:42:26,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-22T03:42:26,523 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c85114ed5096,42545,1732246786482, seqNum=-1] 2024-11-22T03:42:26,523 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=150, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-22T03:42:26,524 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-22T03:42:26,524 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5] 2024-11-22T03:42:26,524 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5] 2024-11-22T03:42:26,525 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37380, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-22T03:42:26,525 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5] 2024-11-22T03:42:26,525 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=150, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-22T03:42:26,525 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37394, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-22T03:42:26,525 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=151, ppid=150, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-22T03:42:26,525 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5] 2024-11-22T03:42:26,526 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5] 2024-11-22T03:42:26,527 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5] 2024-11-22T03:42:26,527 DEBUG [Thread-2184 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-22T03:42:26,528 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c85114ed5096,42545,1732246786482, seqNum=-1] 2024-11-22T03:42:26,528 DEBUG [Thread-2193 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-22T03:42:26,528 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-22T03:42:26,529 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c85114ed5096,42545,1732246786482, seqNum=-1] 2024-11-22T03:42:26,529 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-22T03:42:26,529 DEBUG [Thread-2186 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-22T03:42:26,529 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37400, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-22T03:42:26,530 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37412, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-22T03:42:26,530 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c85114ed5096,42545,1732246786482, seqNum=-1] 2024-11-22T03:42:26,530 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-22T03:42:26,531 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5] 2024-11-22T03:42:26,532 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37416, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-22T03:42:26,532 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5] 2024-11-22T03:42:26,533 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5] 2024-11-22T03:42:26,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(8855): Flush requested on 8422287be02eb151a34ae7fe752afee0 2024-11-22T03:42:26,538 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-22T03:42:26,538 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8422287be02eb151a34ae7fe752afee0 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-22T03:42:26,561 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:42:26,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37384 deadline: 1732247006558, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 2024-11-22T03:42:26,562 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112229a410138d7a4528a77ee0fbb50e27c6_8422287be02eb151a34ae7fe752afee0 is 50, key is test_row_0/A:col10/1732246946527/Put/seqid=0 2024-11-22T03:42:26,562 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:26,562 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:26,562 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:42:26,563 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:42:26,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37416 deadline: 1732247006559, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 2024-11-22T03:42:26,564 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:42:26,564 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:26,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37400 deadline: 1732247006560, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 2024-11-22T03:42:26,564 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:26,564 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:42:26,564 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:26,564 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:26,564 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:42:26,564 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:42:26,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37380 deadline: 1732247006560, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 2024-11-22T03:42:26,564 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:26,564 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:26,564 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:42:26,564 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:42:26,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37382 deadline: 1732247006561, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 2024-11-22T03:42:26,565 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:26,565 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:26,565 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:42:26,577 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742307_1483 (size=12154) 2024-11-22T03:42:26,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-22T03:42:26,669 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:42:26,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37384 deadline: 1732247006668, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 2024-11-22T03:42:26,670 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:26,670 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:42:26,670 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:26,670 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:42:26,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37416 deadline: 1732247006668, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 2024-11-22T03:42:26,670 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:26,670 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:26,670 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:42:26,670 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:42:26,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37380 deadline: 1732247006668, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 2024-11-22T03:42:26,670 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:26,670 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:26,670 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:42:26,670 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:42:26,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37400 deadline: 1732247006668, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 2024-11-22T03:42:26,671 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:26,671 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:26,671 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:42:26,671 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:42:26,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37382 deadline: 1732247006669, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 2024-11-22T03:42:26,671 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:26,671 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:26,671 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:42:26,678 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42545 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-22T03:42:26,678 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0. 2024-11-22T03:42:26,678 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0. as already flushing 2024-11-22T03:42:26,678 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0. 2024-11-22T03:42:26,678 ERROR [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => 8422287be02eb151a34ae7fe752afee0, NAME => 'TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:42:26,678 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => 8422287be02eb151a34ae7fe752afee0, NAME => 'TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:42:26,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42627 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8422287be02eb151a34ae7fe752afee0, NAME => 'TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8422287be02eb151a34ae7fe752afee0, NAME => 'TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:42:26,831 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42545 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-22T03:42:26,831 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0. 2024-11-22T03:42:26,831 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0. as already flushing 2024-11-22T03:42:26,831 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0. 2024-11-22T03:42:26,831 ERROR [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => 8422287be02eb151a34ae7fe752afee0, NAME => 'TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:42:26,831 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => 8422287be02eb151a34ae7fe752afee0, NAME => 'TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:42:26,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42627 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8422287be02eb151a34ae7fe752afee0, NAME => 'TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8422287be02eb151a34ae7fe752afee0, NAME => 'TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:42:26,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-22T03:42:26,880 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:42:26,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37416 deadline: 1732247006878, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 2024-11-22T03:42:26,880 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:42:26,880 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:26,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37380 deadline: 1732247006878, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 2024-11-22T03:42:26,880 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:26,880 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:42:26,881 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:26,881 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:42:26,881 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:26,881 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:42:26,881 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:42:26,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37384 deadline: 1732247006879, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 2024-11-22T03:42:26,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37400 deadline: 1732247006879, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 2024-11-22T03:42:26,881 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:26,881 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:26,881 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:26,881 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:42:26,881 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:42:26,881 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:26,881 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:42:26,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37382 deadline: 1732247006879, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 2024-11-22T03:42:26,881 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:26,881 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:26,881 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:42:26,977 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:42:26,980 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112229a410138d7a4528a77ee0fbb50e27c6_8422287be02eb151a34ae7fe752afee0 to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112229a410138d7a4528a77ee0fbb50e27c6_8422287be02eb151a34ae7fe752afee0 2024-11-22T03:42:26,981 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/.tmp/A/2e8ad2f7b1f74bb9bf07028fb9d30310, store: [table=TestAcidGuarantees family=A region=8422287be02eb151a34ae7fe752afee0] 2024-11-22T03:42:26,981 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/.tmp/A/2e8ad2f7b1f74bb9bf07028fb9d30310 is 175, key is test_row_0/A:col10/1732246946527/Put/seqid=0 2024-11-22T03:42:26,984 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42545 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-22T03:42:26,984 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0. 2024-11-22T03:42:26,984 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0. as already flushing 2024-11-22T03:42:26,984 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0. 2024-11-22T03:42:26,984 ERROR [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => 8422287be02eb151a34ae7fe752afee0, NAME => 'TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:42:26,984 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => 8422287be02eb151a34ae7fe752afee0, NAME => 'TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:42:26,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42627 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8422287be02eb151a34ae7fe752afee0, NAME => 'TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8422287be02eb151a34ae7fe752afee0, NAME => 'TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:42:26,987 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742308_1484 (size=30955) 2024-11-22T03:42:26,987 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=15, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/.tmp/A/2e8ad2f7b1f74bb9bf07028fb9d30310 2024-11-22T03:42:27,007 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/.tmp/B/bc9ddf9d6ddb4cadb7285a33c5637c76 is 50, key is test_row_0/B:col10/1732246946527/Put/seqid=0 2024-11-22T03:42:27,015 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742309_1485 (size=12001) 2024-11-22T03:42:27,136 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42545 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-22T03:42:27,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0. 2024-11-22T03:42:27,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0. as already flushing 2024-11-22T03:42:27,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0. 2024-11-22T03:42:27,137 ERROR [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => 8422287be02eb151a34ae7fe752afee0, NAME => 'TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:42:27,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => 8422287be02eb151a34ae7fe752afee0, NAME => 'TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:42:27,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42627 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8422287be02eb151a34ae7fe752afee0, NAME => 'TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8422287be02eb151a34ae7fe752afee0, NAME => 'TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:42:27,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-22T03:42:27,190 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:42:27,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37400 deadline: 1732247007189, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 2024-11-22T03:42:27,190 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:42:27,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37382 deadline: 1732247007189, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 2024-11-22T03:42:27,191 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:27,191 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:27,191 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:42:27,191 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:27,191 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:27,191 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:42:27,191 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:42:27,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37416 deadline: 1732247007189, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 2024-11-22T03:42:27,191 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:27,191 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:27,191 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:42:27,191 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:42:27,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37384 deadline: 1732247007189, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 2024-11-22T03:42:27,192 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:27,192 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:27,192 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:42:27,192 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:42:27,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37380 deadline: 1732247007189, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 2024-11-22T03:42:27,192 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:27,192 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:27,192 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:42:27,289 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42545 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-22T03:42:27,289 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0. 2024-11-22T03:42:27,289 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0. as already flushing 2024-11-22T03:42:27,289 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0. 2024-11-22T03:42:27,289 ERROR [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => 8422287be02eb151a34ae7fe752afee0, NAME => 'TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:42:27,289 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => 8422287be02eb151a34ae7fe752afee0, NAME => 'TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:42:27,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42627 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8422287be02eb151a34ae7fe752afee0, NAME => 'TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8422287be02eb151a34ae7fe752afee0, NAME => 'TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:42:27,416 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/.tmp/B/bc9ddf9d6ddb4cadb7285a33c5637c76 2024-11-22T03:42:27,434 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/.tmp/C/9b847a0ddb6a411980cdc265c368207b is 50, key is test_row_0/C:col10/1732246946527/Put/seqid=0 2024-11-22T03:42:27,437 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742310_1486 (size=12001) 2024-11-22T03:42:27,442 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42545 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-22T03:42:27,442 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0. 2024-11-22T03:42:27,442 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0. as already flushing 2024-11-22T03:42:27,442 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0. 2024-11-22T03:42:27,442 ERROR [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => 8422287be02eb151a34ae7fe752afee0, NAME => 'TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:42:27,442 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => 8422287be02eb151a34ae7fe752afee0, NAME => 'TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:42:27,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42627 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8422287be02eb151a34ae7fe752afee0, NAME => 'TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8422287be02eb151a34ae7fe752afee0, NAME => 'TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:42:27,594 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42545 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-22T03:42:27,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0. 2024-11-22T03:42:27,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0. as already flushing 2024-11-22T03:42:27,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0. 2024-11-22T03:42:27,595 ERROR [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => 8422287be02eb151a34ae7fe752afee0, NAME => 'TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:42:27,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => 8422287be02eb151a34ae7fe752afee0, NAME => 'TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:42:27,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42627 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8422287be02eb151a34ae7fe752afee0, NAME => 'TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8422287be02eb151a34ae7fe752afee0, NAME => 'TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:42:27,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-22T03:42:27,700 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:42:27,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37400 deadline: 1732247007698, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 2024-11-22T03:42:27,701 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:42:27,701 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:27,701 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:27,701 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:42:27,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37382 deadline: 1732247007698, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 2024-11-22T03:42:27,701 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:27,701 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:42:27,701 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:27,701 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:42:27,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37384 deadline: 1732247007699, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 2024-11-22T03:42:27,701 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:42:27,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37416 deadline: 1732247007699, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 2024-11-22T03:42:27,701 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:27,702 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:27,702 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:42:27,702 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:42:27,702 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:27,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37380 deadline: 1732247007699, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 2024-11-22T03:42:27,702 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:27,702 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:42:27,702 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:27,702 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:27,702 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:42:27,747 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42545 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-22T03:42:27,747 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0. 2024-11-22T03:42:27,747 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0. as already flushing 2024-11-22T03:42:27,747 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0. 2024-11-22T03:42:27,747 ERROR [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => 8422287be02eb151a34ae7fe752afee0, NAME => 'TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:42:27,747 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => 8422287be02eb151a34ae7fe752afee0, NAME => 'TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:42:27,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42627 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8422287be02eb151a34ae7fe752afee0, NAME => 'TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8422287be02eb151a34ae7fe752afee0, NAME => 'TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:42:27,837 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/.tmp/C/9b847a0ddb6a411980cdc265c368207b 2024-11-22T03:42:27,841 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/.tmp/A/2e8ad2f7b1f74bb9bf07028fb9d30310 as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/A/2e8ad2f7b1f74bb9bf07028fb9d30310 2024-11-22T03:42:27,844 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/A/2e8ad2f7b1f74bb9bf07028fb9d30310, entries=150, sequenceid=15, filesize=30.2 K 2024-11-22T03:42:27,844 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/.tmp/B/bc9ddf9d6ddb4cadb7285a33c5637c76 as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/B/bc9ddf9d6ddb4cadb7285a33c5637c76 2024-11-22T03:42:27,847 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/B/bc9ddf9d6ddb4cadb7285a33c5637c76, entries=150, sequenceid=15, filesize=11.7 K 2024-11-22T03:42:27,848 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/.tmp/C/9b847a0ddb6a411980cdc265c368207b as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/C/9b847a0ddb6a411980cdc265c368207b 2024-11-22T03:42:27,851 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/C/9b847a0ddb6a411980cdc265c368207b, entries=150, sequenceid=15, filesize=11.7 K 2024-11-22T03:42:27,851 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 8422287be02eb151a34ae7fe752afee0 in 1313ms, sequenceid=15, compaction requested=false 2024-11-22T03:42:27,851 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8422287be02eb151a34ae7fe752afee0: 2024-11-22T03:42:27,901 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42545 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-22T03:42:27,901 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0. 2024-11-22T03:42:27,901 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2902): Flushing 8422287be02eb151a34ae7fe752afee0 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-22T03:42:27,909 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411227bb911e2d0cd4413a8a73683153faed0_8422287be02eb151a34ae7fe752afee0 is 50, key is test_row_0/A:col10/1732246946559/Put/seqid=0 2024-11-22T03:42:27,912 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742311_1487 (size=12154) 2024-11-22T03:42:27,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:42:27,917 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411227bb911e2d0cd4413a8a73683153faed0_8422287be02eb151a34ae7fe752afee0 to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411227bb911e2d0cd4413a8a73683153faed0_8422287be02eb151a34ae7fe752afee0 2024-11-22T03:42:27,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/.tmp/A/d14c9c77ce1a494da6bfea3849d1f037, store: [table=TestAcidGuarantees family=A region=8422287be02eb151a34ae7fe752afee0] 2024-11-22T03:42:27,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/.tmp/A/d14c9c77ce1a494da6bfea3849d1f037 is 175, key is test_row_0/A:col10/1732246946559/Put/seqid=0 2024-11-22T03:42:27,930 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742312_1488 (size=30955) 2024-11-22T03:42:28,129 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-22T03:42:28,334 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=40, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/.tmp/A/d14c9c77ce1a494da6bfea3849d1f037 2024-11-22T03:42:28,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/.tmp/B/4d255fc3ee3c47de92044e1ad0b0fac3 is 50, key is test_row_0/B:col10/1732246946559/Put/seqid=0 2024-11-22T03:42:28,353 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742313_1489 (size=12001) 2024-11-22T03:42:28,353 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=40 (bloomFilter=true), to=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/.tmp/B/4d255fc3ee3c47de92044e1ad0b0fac3 2024-11-22T03:42:28,366 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/.tmp/C/60c175ad58b343a8b05a41fd91fa939c is 50, key is test_row_0/C:col10/1732246946559/Put/seqid=0 2024-11-22T03:42:28,378 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742314_1490 (size=12001) 2024-11-22T03:42:28,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-22T03:42:28,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(8855): Flush requested on 8422287be02eb151a34ae7fe752afee0 2024-11-22T03:42:28,709 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0. as already flushing 2024-11-22T03:42:28,743 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:42:28,743 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:42:28,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37384 deadline: 1732247008723, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 2024-11-22T03:42:28,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37380 deadline: 1732247008719, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 2024-11-22T03:42:28,744 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:28,744 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:28,744 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:42:28,744 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to c85114ed5096,42545,1732246786482 for 'test_row_1' in 8422287be02eb151a34ae7fe752afee0 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2183 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-22T03:42:28,744 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:28,744 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:28,744 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:42:28,747 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:42:28,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37382 deadline: 1732247008743, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 2024-11-22T03:42:28,748 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:42:28,748 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:28,748 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:28,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37416 deadline: 1732247008743, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 2024-11-22T03:42:28,748 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:42:28,748 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:42:28,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37400 deadline: 1732247008743, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 2024-11-22T03:42:28,748 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:28,748 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:28,748 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:42:28,749 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:28,749 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:28,749 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:42:28,779 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=40 (bloomFilter=true), to=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/.tmp/C/60c175ad58b343a8b05a41fd91fa939c 2024-11-22T03:42:28,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/.tmp/A/d14c9c77ce1a494da6bfea3849d1f037 as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/A/d14c9c77ce1a494da6bfea3849d1f037 2024-11-22T03:42:28,787 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/A/d14c9c77ce1a494da6bfea3849d1f037, entries=150, sequenceid=40, filesize=30.2 K 2024-11-22T03:42:28,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/.tmp/B/4d255fc3ee3c47de92044e1ad0b0fac3 as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/B/4d255fc3ee3c47de92044e1ad0b0fac3 2024-11-22T03:42:28,791 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/B/4d255fc3ee3c47de92044e1ad0b0fac3, entries=150, sequenceid=40, filesize=11.7 K 2024-11-22T03:42:28,792 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/.tmp/C/60c175ad58b343a8b05a41fd91fa939c as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/C/60c175ad58b343a8b05a41fd91fa939c 2024-11-22T03:42:28,798 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/C/60c175ad58b343a8b05a41fd91fa939c, entries=150, sequenceid=40, filesize=11.7 K 2024-11-22T03:42:28,799 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 8422287be02eb151a34ae7fe752afee0 in 898ms, sequenceid=40, compaction requested=false 2024-11-22T03:42:28,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2603): Flush status journal for 8422287be02eb151a34ae7fe752afee0: 2024-11-22T03:42:28,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0. 2024-11-22T03:42:28,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=151 2024-11-22T03:42:28,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42627 {}] master.HMaster(4169): Remote procedure done, pid=151 2024-11-22T03:42:28,802 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=151, resume processing ppid=150 2024-11-22T03:42:28,802 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=151, ppid=150, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2750 sec 2024-11-22T03:42:28,804 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=150, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees in 2.2820 sec 2024-11-22T03:42:28,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(8855): Flush requested on 8422287be02eb151a34ae7fe752afee0 2024-11-22T03:42:28,851 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-22T03:42:28,851 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8422287be02eb151a34ae7fe752afee0 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-22T03:42:28,856 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112203a74c5561ae41b49ba11e199b7c0f5f_8422287be02eb151a34ae7fe752afee0 is 50, key is test_row_0/A:col10/1732246948730/Put/seqid=0 2024-11-22T03:42:28,878 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742315_1491 (size=12154) 2024-11-22T03:42:28,890 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:42:28,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37416 deadline: 1732247008884, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 2024-11-22T03:42:28,890 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:42:28,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37384 deadline: 1732247008886, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 2024-11-22T03:42:28,890 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:28,890 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:28,890 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:42:28,890 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:28,890 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:28,890 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:42:28,893 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:42:28,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37400 deadline: 1732247008890, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 2024-11-22T03:42:28,894 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:28,894 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:42:28,894 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:28,894 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:42:28,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37382 deadline: 1732247008890, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 2024-11-22T03:42:28,894 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:28,894 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:28,895 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:42:29,001 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:42:29,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37384 deadline: 1732247008999, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 2024-11-22T03:42:29,001 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:42:29,001 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:29,001 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:29,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37416 deadline: 1732247008999, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 2024-11-22T03:42:29,001 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:42:29,002 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:42:29,002 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:29,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37382 deadline: 1732247008999, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 2024-11-22T03:42:29,002 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:29,002 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:42:29,002 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:29,002 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:42:29,002 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:29,002 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:42:29,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37400 deadline: 1732247008999, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 2024-11-22T03:42:29,002 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:29,002 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:29,002 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:42:29,210 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:42:29,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37384 deadline: 1732247009208, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 2024-11-22T03:42:29,211 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:29,211 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:42:29,211 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:29,211 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:42:29,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37416 deadline: 1732247009208, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 2024-11-22T03:42:29,211 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:29,211 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:29,211 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:42:29,211 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:42:29,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37382 deadline: 1732247009208, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 2024-11-22T03:42:29,211 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:29,212 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:29,212 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:42:29,212 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:42:29,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37400 deadline: 1732247009209, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 2024-11-22T03:42:29,212 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:29,212 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:29,212 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:42:29,279 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:42:29,281 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112203a74c5561ae41b49ba11e199b7c0f5f_8422287be02eb151a34ae7fe752afee0 to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112203a74c5561ae41b49ba11e199b7c0f5f_8422287be02eb151a34ae7fe752afee0 2024-11-22T03:42:29,282 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/.tmp/A/938c4314a44746aba24bca9f781823e2, store: [table=TestAcidGuarantees family=A region=8422287be02eb151a34ae7fe752afee0] 2024-11-22T03:42:29,282 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/.tmp/A/938c4314a44746aba24bca9f781823e2 is 175, key is test_row_0/A:col10/1732246948730/Put/seqid=0 2024-11-22T03:42:29,285 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742316_1492 (size=30955) 2024-11-22T03:42:29,286 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=52, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/.tmp/A/938c4314a44746aba24bca9f781823e2 2024-11-22T03:42:29,291 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/.tmp/B/4104be0e041f43308af292e753ff368c is 50, key is test_row_0/B:col10/1732246948730/Put/seqid=0 2024-11-22T03:42:29,295 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742317_1493 (size=12001) 2024-11-22T03:42:29,519 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:42:29,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37384 deadline: 1732247009518, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 2024-11-22T03:42:29,520 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:42:29,520 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:29,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37416 deadline: 1732247009518, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 2024-11-22T03:42:29,520 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:29,520 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:42:29,520 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:42:29,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37382 deadline: 1732247009518, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 2024-11-22T03:42:29,520 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:29,520 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:29,520 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:42:29,520 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:29,520 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:42:29,520 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:29,520 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:42:29,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37400 deadline: 1732247009518, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 2024-11-22T03:42:29,520 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:29,521 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:29,521 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:42:29,695 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=52 (bloomFilter=true), to=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/.tmp/B/4104be0e041f43308af292e753ff368c 2024-11-22T03:42:29,700 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/.tmp/C/913085e92be04ce0a3138e7bd619645d is 50, key is test_row_0/C:col10/1732246948730/Put/seqid=0 2024-11-22T03:42:29,703 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742318_1494 (size=12001) 2024-11-22T03:42:30,031 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:42:30,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37384 deadline: 1732247010028, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 2024-11-22T03:42:30,031 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:30,031 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:30,031 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:42:30,031 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:42:30,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37382 deadline: 1732247010028, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 2024-11-22T03:42:30,032 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:30,032 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:30,032 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:42:30,032 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:42:30,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37416 deadline: 1732247010028, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 2024-11-22T03:42:30,032 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:42:30,032 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:30,032 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:30,032 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:42:30,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37400 deadline: 1732247010029, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 2024-11-22T03:42:30,032 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:30,032 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:30,032 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:42:30,104 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=52 (bloomFilter=true), to=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/.tmp/C/913085e92be04ce0a3138e7bd619645d 2024-11-22T03:42:30,107 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/.tmp/A/938c4314a44746aba24bca9f781823e2 as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/A/938c4314a44746aba24bca9f781823e2 2024-11-22T03:42:30,109 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/A/938c4314a44746aba24bca9f781823e2, entries=150, sequenceid=52, filesize=30.2 K 2024-11-22T03:42:30,110 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/.tmp/B/4104be0e041f43308af292e753ff368c as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/B/4104be0e041f43308af292e753ff368c 2024-11-22T03:42:30,113 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/B/4104be0e041f43308af292e753ff368c, entries=150, sequenceid=52, filesize=11.7 K 2024-11-22T03:42:30,114 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/.tmp/C/913085e92be04ce0a3138e7bd619645d as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/C/913085e92be04ce0a3138e7bd619645d 2024-11-22T03:42:30,117 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/C/913085e92be04ce0a3138e7bd619645d, entries=150, sequenceid=52, filesize=11.7 K 2024-11-22T03:42:30,118 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 8422287be02eb151a34ae7fe752afee0 in 1266ms, sequenceid=52, compaction requested=true 2024-11-22T03:42:30,118 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8422287be02eb151a34ae7fe752afee0: 2024-11-22T03:42:30,118 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8422287be02eb151a34ae7fe752afee0:A, priority=-2147483648, current under compaction store size is 1 2024-11-22T03:42:30,118 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T03:42:30,118 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8422287be02eb151a34ae7fe752afee0:B, priority=-2147483648, current under compaction store size is 2 2024-11-22T03:42:30,118 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T03:42:30,118 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T03:42:30,118 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T03:42:30,118 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8422287be02eb151a34ae7fe752afee0:C, priority=-2147483648, current under compaction store size is 3 2024-11-22T03:42:30,118 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T03:42:30,119 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 92865 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T03:42:30,119 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.HStore(1541): 8422287be02eb151a34ae7fe752afee0/A is initiating minor compaction (all files) 2024-11-22T03:42:30,119 INFO [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8422287be02eb151a34ae7fe752afee0/A in TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0. 2024-11-22T03:42:30,119 INFO [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/A/2e8ad2f7b1f74bb9bf07028fb9d30310, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/A/d14c9c77ce1a494da6bfea3849d1f037, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/A/938c4314a44746aba24bca9f781823e2] into tmpdir=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/.tmp, totalSize=90.7 K 2024-11-22T03:42:30,119 INFO [RS:0;c85114ed5096:42545-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0. 2024-11-22T03:42:30,119 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0. files: [hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/A/2e8ad2f7b1f74bb9bf07028fb9d30310, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/A/d14c9c77ce1a494da6bfea3849d1f037, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/A/938c4314a44746aba24bca9f781823e2] 2024-11-22T03:42:30,119 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T03:42:30,119 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2e8ad2f7b1f74bb9bf07028fb9d30310, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1732246946527 2024-11-22T03:42:30,119 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HStore(1541): 8422287be02eb151a34ae7fe752afee0/B is initiating minor compaction (all files) 2024-11-22T03:42:30,119 INFO [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8422287be02eb151a34ae7fe752afee0/B in TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0. 2024-11-22T03:42:30,119 INFO [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/B/bc9ddf9d6ddb4cadb7285a33c5637c76, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/B/4d255fc3ee3c47de92044e1ad0b0fac3, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/B/4104be0e041f43308af292e753ff368c] into tmpdir=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/.tmp, totalSize=35.2 K 2024-11-22T03:42:30,120 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] compactions.Compactor(225): Compacting d14c9c77ce1a494da6bfea3849d1f037, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1732246946553 2024-11-22T03:42:30,120 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] compactions.Compactor(225): Compacting bc9ddf9d6ddb4cadb7285a33c5637c76, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1732246946527 2024-11-22T03:42:30,120 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] compactions.Compactor(225): Compacting 4d255fc3ee3c47de92044e1ad0b0fac3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1732246946553 2024-11-22T03:42:30,120 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] compactions.Compactor(225): Compacting 938c4314a44746aba24bca9f781823e2, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1732246948714 2024-11-22T03:42:30,120 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] compactions.Compactor(225): Compacting 4104be0e041f43308af292e753ff368c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1732246948714 2024-11-22T03:42:30,126 INFO [RS:0;c85114ed5096:42545-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=8422287be02eb151a34ae7fe752afee0] 2024-11-22T03:42:30,128 INFO [RS:0;c85114ed5096:42545-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8422287be02eb151a34ae7fe752afee0#B#compaction#424 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-22T03:42:30,128 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/.tmp/B/d85467796c174b3b9d6176079597e3cf is 50, key is test_row_0/B:col10/1732246948730/Put/seqid=0 2024-11-22T03:42:30,133 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411226eaaec8dc0184e21817447eb6ea672ee_8422287be02eb151a34ae7fe752afee0 store=[table=TestAcidGuarantees family=A region=8422287be02eb151a34ae7fe752afee0] 2024-11-22T03:42:30,136 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411226eaaec8dc0184e21817447eb6ea672ee_8422287be02eb151a34ae7fe752afee0, store=[table=TestAcidGuarantees family=A region=8422287be02eb151a34ae7fe752afee0] 2024-11-22T03:42:30,136 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411226eaaec8dc0184e21817447eb6ea672ee_8422287be02eb151a34ae7fe752afee0 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=8422287be02eb151a34ae7fe752afee0] 2024-11-22T03:42:30,146 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742319_1495 (size=12104) 2024-11-22T03:42:30,154 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742320_1496 (size=4469) 2024-11-22T03:42:30,159 INFO [RS:0;c85114ed5096:42545-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8422287be02eb151a34ae7fe752afee0#A#compaction#423 average throughput is 0.74 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T03:42:30,159 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/.tmp/A/84c77362f3a8479ba4a6997ea26478a5 is 175, key is test_row_0/A:col10/1732246948730/Put/seqid=0 2024-11-22T03:42:30,162 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742321_1497 (size=31058) 2024-11-22T03:42:30,550 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/.tmp/B/d85467796c174b3b9d6176079597e3cf as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/B/d85467796c174b3b9d6176079597e3cf 2024-11-22T03:42:30,553 INFO [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8422287be02eb151a34ae7fe752afee0/B of 8422287be02eb151a34ae7fe752afee0 into d85467796c174b3b9d6176079597e3cf(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T03:42:30,554 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8422287be02eb151a34ae7fe752afee0: 2024-11-22T03:42:30,554 INFO [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., storeName=8422287be02eb151a34ae7fe752afee0/B, priority=13, startTime=1732246950118; duration=0sec 2024-11-22T03:42:30,554 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T03:42:30,554 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8422287be02eb151a34ae7fe752afee0:B 2024-11-22T03:42:30,554 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T03:42:30,555 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T03:42:30,555 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HStore(1541): 8422287be02eb151a34ae7fe752afee0/C is initiating minor compaction (all files) 2024-11-22T03:42:30,555 INFO [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8422287be02eb151a34ae7fe752afee0/C in TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0. 2024-11-22T03:42:30,555 INFO [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/C/9b847a0ddb6a411980cdc265c368207b, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/C/60c175ad58b343a8b05a41fd91fa939c, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/C/913085e92be04ce0a3138e7bd619645d] into tmpdir=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/.tmp, totalSize=35.2 K 2024-11-22T03:42:30,556 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] compactions.Compactor(225): Compacting 9b847a0ddb6a411980cdc265c368207b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1732246946527 2024-11-22T03:42:30,556 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] compactions.Compactor(225): Compacting 60c175ad58b343a8b05a41fd91fa939c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1732246946553 2024-11-22T03:42:30,556 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] compactions.Compactor(225): Compacting 913085e92be04ce0a3138e7bd619645d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1732246948714 2024-11-22T03:42:30,564 INFO [RS:0;c85114ed5096:42545-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8422287be02eb151a34ae7fe752afee0#C#compaction#425 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T03:42:30,565 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/.tmp/C/6a403ba20043415fa48677d75ee90ed6 is 50, key is test_row_0/C:col10/1732246948730/Put/seqid=0 2024-11-22T03:42:30,570 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/.tmp/A/84c77362f3a8479ba4a6997ea26478a5 as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/A/84c77362f3a8479ba4a6997ea26478a5 2024-11-22T03:42:30,571 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742322_1498 (size=12104) 2024-11-22T03:42:30,574 INFO [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8422287be02eb151a34ae7fe752afee0/A of 8422287be02eb151a34ae7fe752afee0 into 84c77362f3a8479ba4a6997ea26478a5(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T03:42:30,574 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8422287be02eb151a34ae7fe752afee0: 2024-11-22T03:42:30,574 INFO [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., storeName=8422287be02eb151a34ae7fe752afee0/A, priority=13, startTime=1732246950118; duration=0sec 2024-11-22T03:42:30,574 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T03:42:30,574 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8422287be02eb151a34ae7fe752afee0:A 2024-11-22T03:42:30,585 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/.tmp/C/6a403ba20043415fa48677d75ee90ed6 as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/C/6a403ba20043415fa48677d75ee90ed6 2024-11-22T03:42:30,588 INFO [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8422287be02eb151a34ae7fe752afee0/C of 8422287be02eb151a34ae7fe752afee0 into 6a403ba20043415fa48677d75ee90ed6(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T03:42:30,589 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8422287be02eb151a34ae7fe752afee0: 2024-11-22T03:42:30,589 INFO [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., storeName=8422287be02eb151a34ae7fe752afee0/C, priority=13, startTime=1732246950118; duration=0sec 2024-11-22T03:42:30,589 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T03:42:30,589 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8422287be02eb151a34ae7fe752afee0:C 2024-11-22T03:42:30,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-22T03:42:30,688 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-22T03:42:30,689 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42627 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-22T03:42:30,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42627 {}] procedure2.ProcedureExecutor(1139): Stored pid=152, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees 2024-11-22T03:42:30,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-22T03:42:30,690 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=152, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-22T03:42:30,691 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=152, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-22T03:42:30,691 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=153, ppid=152, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-22T03:42:30,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(8855): Flush requested on 8422287be02eb151a34ae7fe752afee0 2024-11-22T03:42:30,771 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-22T03:42:30,771 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8422287be02eb151a34ae7fe752afee0 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-22T03:42:30,778 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241122410292eb2ce4420dadfcca6799089867_8422287be02eb151a34ae7fe752afee0 is 50, key is test_row_0/A:col10/1732246948886/Put/seqid=0 2024-11-22T03:42:30,781 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742323_1499 (size=14594) 2024-11-22T03:42:30,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-22T03:42:30,801 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:42:30,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37380 deadline: 1732247010799, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 2024-11-22T03:42:30,802 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:30,802 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:30,802 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:42:30,843 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42545 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-22T03:42:30,844 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0. 2024-11-22T03:42:30,844 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0. as already flushing 2024-11-22T03:42:30,844 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0. 2024-11-22T03:42:30,844 ERROR [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 8422287be02eb151a34ae7fe752afee0, NAME => 'TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:42:30,844 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 8422287be02eb151a34ae7fe752afee0, NAME => 'TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:42:30,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42627 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8422287be02eb151a34ae7fe752afee0, NAME => 'TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8422287be02eb151a34ae7fe752afee0, NAME => 'TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:42:30,911 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:42:30,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37380 deadline: 1732247010908, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 2024-11-22T03:42:30,911 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:30,911 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:30,912 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:42:30,996 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42545 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-22T03:42:30,996 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0. 2024-11-22T03:42:30,996 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0. as already flushing 2024-11-22T03:42:30,996 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0. 2024-11-22T03:42:30,996 ERROR [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 8422287be02eb151a34ae7fe752afee0, NAME => 'TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:42:30,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 8422287be02eb151a34ae7fe752afee0, NAME => 'TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:42:30,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42627 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8422287be02eb151a34ae7fe752afee0, NAME => 'TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8422287be02eb151a34ae7fe752afee0, NAME => 'TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:42:31,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-22T03:42:31,039 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:42:31,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37384 deadline: 1732247011038, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 2024-11-22T03:42:31,040 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:42:31,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37400 deadline: 1732247011038, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 2024-11-22T03:42:31,040 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to c85114ed5096,42545,1732246786482 for 'test_row_1' in 8422287be02eb151a34ae7fe752afee0 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2154 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-22T03:42:31,040 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:31,040 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:31,040 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:42:31,040 WARN [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRpcRetryingCaller(177): Call to c85114ed5096,42545,1732246786482 for 'test_row_1' in 8422287be02eb151a34ae7fe752afee0 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2150 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-22T03:42:31,040 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:31,040 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:31,040 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:42:31,051 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:42:31,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37382 deadline: 1732247011048, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 2024-11-22T03:42:31,052 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:42:31,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37416 deadline: 1732247011049, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 2024-11-22T03:42:31,052 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to c85114ed5096,42545,1732246786482 for 'test_row_2' in 8422287be02eb151a34ae7fe752afee0 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2161 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-22T03:42:31,052 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:31,052 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:31,052 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:42:31,052 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to c85114ed5096,42545,1732246786482 for 'test_row_0' in 8422287be02eb151a34ae7fe752afee0 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2168 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-22T03:42:31,053 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:31,053 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:31,053 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:42:31,119 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:42:31,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37380 deadline: 1732247011118, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 2024-11-22T03:42:31,120 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:31,120 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:31,120 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:42:31,149 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42545 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-22T03:42:31,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0. 2024-11-22T03:42:31,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0. as already flushing 2024-11-22T03:42:31,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0. 2024-11-22T03:42:31,149 ERROR [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 8422287be02eb151a34ae7fe752afee0, NAME => 'TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:42:31,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 8422287be02eb151a34ae7fe752afee0, NAME => 'TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:42:31,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42627 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8422287be02eb151a34ae7fe752afee0, NAME => 'TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8422287be02eb151a34ae7fe752afee0, NAME => 'TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:42:31,182 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:42:31,185 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241122410292eb2ce4420dadfcca6799089867_8422287be02eb151a34ae7fe752afee0 to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241122410292eb2ce4420dadfcca6799089867_8422287be02eb151a34ae7fe752afee0 2024-11-22T03:42:31,186 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/.tmp/A/f58c125a588e45e08433dff8062a916c, store: [table=TestAcidGuarantees family=A region=8422287be02eb151a34ae7fe752afee0] 2024-11-22T03:42:31,186 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/.tmp/A/f58c125a588e45e08433dff8062a916c is 175, key is test_row_0/A:col10/1732246948886/Put/seqid=0 2024-11-22T03:42:31,189 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742324_1500 (size=39549) 2024-11-22T03:42:31,190 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=80, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/.tmp/A/f58c125a588e45e08433dff8062a916c 2024-11-22T03:42:31,196 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/.tmp/B/7c04c26321b94ba8b67a3c48d13e214e is 50, key is test_row_0/B:col10/1732246948886/Put/seqid=0 2024-11-22T03:42:31,204 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742325_1501 (size=12001) 2024-11-22T03:42:31,301 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42545 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-22T03:42:31,301 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0. 2024-11-22T03:42:31,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0. as already flushing 2024-11-22T03:42:31,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0. 2024-11-22T03:42:31,302 ERROR [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 8422287be02eb151a34ae7fe752afee0, NAME => 'TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:42:31,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 8422287be02eb151a34ae7fe752afee0, NAME => 'TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:42:31,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42627 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8422287be02eb151a34ae7fe752afee0, NAME => 'TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8422287be02eb151a34ae7fe752afee0, NAME => 'TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:42:31,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-22T03:42:31,431 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:42:31,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37380 deadline: 1732247011429, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 2024-11-22T03:42:31,432 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:31,432 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:31,432 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:42:31,454 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42545 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-22T03:42:31,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0. 2024-11-22T03:42:31,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0. as already flushing 2024-11-22T03:42:31,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0. 2024-11-22T03:42:31,454 ERROR [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 8422287be02eb151a34ae7fe752afee0, NAME => 'TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:42:31,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 8422287be02eb151a34ae7fe752afee0, NAME => 'TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:42:31,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42627 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8422287be02eb151a34ae7fe752afee0, NAME => 'TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8422287be02eb151a34ae7fe752afee0, NAME => 'TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:42:31,605 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=80 (bloomFilter=true), to=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/.tmp/B/7c04c26321b94ba8b67a3c48d13e214e 2024-11-22T03:42:31,606 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42545 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-22T03:42:31,607 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0. 2024-11-22T03:42:31,607 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0. as already flushing 2024-11-22T03:42:31,607 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0. 2024-11-22T03:42:31,607 ERROR [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 8422287be02eb151a34ae7fe752afee0, NAME => 'TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:42:31,607 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 8422287be02eb151a34ae7fe752afee0, NAME => 'TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:42:31,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42627 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8422287be02eb151a34ae7fe752afee0, NAME => 'TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8422287be02eb151a34ae7fe752afee0, NAME => 'TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:42:31,610 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/.tmp/C/f9056aac4dc740059dd3efb37f86d910 is 50, key is test_row_0/C:col10/1732246948886/Put/seqid=0 2024-11-22T03:42:31,614 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742326_1502 (size=12001) 2024-11-22T03:42:31,759 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42545 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-22T03:42:31,759 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0. 2024-11-22T03:42:31,759 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0. as already flushing 2024-11-22T03:42:31,760 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0. 2024-11-22T03:42:31,760 ERROR [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 8422287be02eb151a34ae7fe752afee0, NAME => 'TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:42:31,760 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 8422287be02eb151a34ae7fe752afee0, NAME => 'TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:42:31,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42627 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8422287be02eb151a34ae7fe752afee0, NAME => 'TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8422287be02eb151a34ae7fe752afee0, NAME => 'TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:42:31,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-22T03:42:31,912 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42545 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-22T03:42:31,912 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0. 2024-11-22T03:42:31,912 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0. as already flushing 2024-11-22T03:42:31,912 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0. 2024-11-22T03:42:31,912 ERROR [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 8422287be02eb151a34ae7fe752afee0, NAME => 'TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:42:31,912 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 8422287be02eb151a34ae7fe752afee0, NAME => 'TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:42:31,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42627 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8422287be02eb151a34ae7fe752afee0, NAME => 'TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8422287be02eb151a34ae7fe752afee0, NAME => 'TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:42:31,940 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:42:31,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37380 deadline: 1732247011938, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 2024-11-22T03:42:31,940 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:31,940 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:31,940 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:42:32,015 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=80 (bloomFilter=true), to=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/.tmp/C/f9056aac4dc740059dd3efb37f86d910 2024-11-22T03:42:32,018 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/.tmp/A/f58c125a588e45e08433dff8062a916c as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/A/f58c125a588e45e08433dff8062a916c 2024-11-22T03:42:32,021 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/A/f58c125a588e45e08433dff8062a916c, entries=200, sequenceid=80, filesize=38.6 K 2024-11-22T03:42:32,022 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/.tmp/B/7c04c26321b94ba8b67a3c48d13e214e as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/B/7c04c26321b94ba8b67a3c48d13e214e 2024-11-22T03:42:32,025 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/B/7c04c26321b94ba8b67a3c48d13e214e, entries=150, sequenceid=80, filesize=11.7 K 2024-11-22T03:42:32,025 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/.tmp/C/f9056aac4dc740059dd3efb37f86d910 as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/C/f9056aac4dc740059dd3efb37f86d910 2024-11-22T03:42:32,028 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/C/f9056aac4dc740059dd3efb37f86d910, entries=150, sequenceid=80, filesize=11.7 K 2024-11-22T03:42:32,029 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 8422287be02eb151a34ae7fe752afee0 in 1258ms, sequenceid=80, compaction requested=false 2024-11-22T03:42:32,029 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8422287be02eb151a34ae7fe752afee0: 2024-11-22T03:42:32,064 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42545 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-22T03:42:32,065 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0. 2024-11-22T03:42:32,065 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2902): Flushing 8422287be02eb151a34ae7fe752afee0 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-22T03:42:32,072 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411222683af9cf0304abdbb554b88a9cbf243_8422287be02eb151a34ae7fe752afee0 is 50, key is test_row_0/A:col10/1732246950794/Put/seqid=0 2024-11-22T03:42:32,075 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742327_1503 (size=12154) 2024-11-22T03:42:32,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:42:32,478 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411222683af9cf0304abdbb554b88a9cbf243_8422287be02eb151a34ae7fe752afee0 to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411222683af9cf0304abdbb554b88a9cbf243_8422287be02eb151a34ae7fe752afee0 2024-11-22T03:42:32,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/.tmp/A/08d77b7752a94b26971efbb82978da3b, store: [table=TestAcidGuarantees family=A region=8422287be02eb151a34ae7fe752afee0] 2024-11-22T03:42:32,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/.tmp/A/08d77b7752a94b26971efbb82978da3b is 175, key is test_row_0/A:col10/1732246950794/Put/seqid=0 2024-11-22T03:42:32,483 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742328_1504 (size=30955) 2024-11-22T03:42:32,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-22T03:42:32,884 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=91, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/.tmp/A/08d77b7752a94b26971efbb82978da3b 2024-11-22T03:42:32,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/.tmp/B/276fa3c198bf43f0af4cdee49a168057 is 50, key is test_row_0/B:col10/1732246950794/Put/seqid=0 2024-11-22T03:42:32,891 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742329_1505 (size=12001) 2024-11-22T03:42:32,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(8855): Flush requested on 8422287be02eb151a34ae7fe752afee0 2024-11-22T03:42:32,951 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0. as already flushing 2024-11-22T03:42:33,036 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:42:33,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37380 deadline: 1732247013033, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 2024-11-22T03:42:33,037 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:33,037 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:33,037 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:42:33,050 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:42:33,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37400 deadline: 1732247013049, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 2024-11-22T03:42:33,051 WARN [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRpcRetryingCaller(177): Call to c85114ed5096,42545,1732246786482 for 'test_row_1' in 8422287be02eb151a34ae7fe752afee0 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4160 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-22T03:42:33,051 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:33,051 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:33,051 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:42:33,059 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:42:33,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37384 deadline: 1732247013058, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 2024-11-22T03:42:33,059 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to c85114ed5096,42545,1732246786482 for 'test_row_1' in 8422287be02eb151a34ae7fe752afee0 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4173 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-22T03:42:33,059 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:33,059 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:33,059 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:42:33,071 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:42:33,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37382 deadline: 1732247013068, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 2024-11-22T03:42:33,071 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:42:33,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37416 deadline: 1732247013068, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 2024-11-22T03:42:33,071 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to c85114ed5096,42545,1732246786482 for 'test_row_2' in 8422287be02eb151a34ae7fe752afee0 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4181 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-22T03:42:33,071 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to c85114ed5096,42545,1732246786482 for 'test_row_0' in 8422287be02eb151a34ae7fe752afee0 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4187 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-22T03:42:33,072 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:33,072 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:33,072 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:33,072 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:33,072 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:42:33,072 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:42:33,140 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:42:33,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37380 deadline: 1732247013138, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 2024-11-22T03:42:33,141 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:33,141 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:33,141 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:42:33,292 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=91 (bloomFilter=true), to=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/.tmp/B/276fa3c198bf43f0af4cdee49a168057 2024-11-22T03:42:33,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/.tmp/C/f370abdfa2174d93bdfd2318f0c74cb7 is 50, key is test_row_0/C:col10/1732246950794/Put/seqid=0 2024-11-22T03:42:33,311 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742330_1506 (size=12001) 2024-11-22T03:42:33,312 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=91 (bloomFilter=true), to=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/.tmp/C/f370abdfa2174d93bdfd2318f0c74cb7 2024-11-22T03:42:33,315 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/.tmp/A/08d77b7752a94b26971efbb82978da3b as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/A/08d77b7752a94b26971efbb82978da3b 2024-11-22T03:42:33,318 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/A/08d77b7752a94b26971efbb82978da3b, entries=150, sequenceid=91, filesize=30.2 K 2024-11-22T03:42:33,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/.tmp/B/276fa3c198bf43f0af4cdee49a168057 as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/B/276fa3c198bf43f0af4cdee49a168057 2024-11-22T03:42:33,322 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/B/276fa3c198bf43f0af4cdee49a168057, entries=150, sequenceid=91, filesize=11.7 K 2024-11-22T03:42:33,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/.tmp/C/f370abdfa2174d93bdfd2318f0c74cb7 as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/C/f370abdfa2174d93bdfd2318f0c74cb7 2024-11-22T03:42:33,326 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/C/f370abdfa2174d93bdfd2318f0c74cb7, entries=150, sequenceid=91, filesize=11.7 K 2024-11-22T03:42:33,327 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 8422287be02eb151a34ae7fe752afee0 in 1262ms, sequenceid=91, compaction requested=true 2024-11-22T03:42:33,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2603): Flush status journal for 8422287be02eb151a34ae7fe752afee0: 2024-11-22T03:42:33,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0. 2024-11-22T03:42:33,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=153 2024-11-22T03:42:33,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42627 {}] master.HMaster(4169): Remote procedure done, pid=153 2024-11-22T03:42:33,329 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=153, resume processing ppid=152 2024-11-22T03:42:33,329 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=153, ppid=152, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.6370 sec 2024-11-22T03:42:33,331 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=152, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees in 2.6410 sec 2024-11-22T03:42:33,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(8855): Flush requested on 8422287be02eb151a34ae7fe752afee0 2024-11-22T03:42:33,351 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-22T03:42:33,351 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8422287be02eb151a34ae7fe752afee0 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-22T03:42:33,358 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411223dade7110c8d442ba6a0816b290ae6e1_8422287be02eb151a34ae7fe752afee0 is 50, key is test_row_0/A:col10/1732246953031/Put/seqid=0 2024-11-22T03:42:33,361 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742331_1507 (size=14594) 2024-11-22T03:42:33,384 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:42:33,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37380 deadline: 1732247013381, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 2024-11-22T03:42:33,385 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:33,385 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:33,385 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:42:33,491 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:42:33,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37380 deadline: 1732247013488, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 2024-11-22T03:42:33,491 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:33,491 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:33,491 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:42:33,700 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:42:33,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37380 deadline: 1732247013698, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 2024-11-22T03:42:33,700 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:33,700 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:33,700 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:42:33,761 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:42:33,764 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411223dade7110c8d442ba6a0816b290ae6e1_8422287be02eb151a34ae7fe752afee0 to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411223dade7110c8d442ba6a0816b290ae6e1_8422287be02eb151a34ae7fe752afee0 2024-11-22T03:42:33,764 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/.tmp/A/f3a5993a5b7a4d7baccc7a9b2f3d5aaf, store: [table=TestAcidGuarantees family=A region=8422287be02eb151a34ae7fe752afee0] 2024-11-22T03:42:33,765 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/.tmp/A/f3a5993a5b7a4d7baccc7a9b2f3d5aaf is 175, key is test_row_0/A:col10/1732246953031/Put/seqid=0 2024-11-22T03:42:33,768 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742332_1508 (size=39549) 2024-11-22T03:42:34,011 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:42:34,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37380 deadline: 1732247014008, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 2024-11-22T03:42:34,011 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:34,011 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:34,011 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:42:34,169 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=117, memsize=51.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/.tmp/A/f3a5993a5b7a4d7baccc7a9b2f3d5aaf 2024-11-22T03:42:34,175 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/.tmp/B/95ac750810af44f6b683b180c5c55c4d is 50, key is test_row_0/B:col10/1732246953031/Put/seqid=0 2024-11-22T03:42:34,178 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742333_1509 (size=12001) 2024-11-22T03:42:34,186 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=117 (bloomFilter=true), to=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/.tmp/B/95ac750810af44f6b683b180c5c55c4d 2024-11-22T03:42:34,192 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/.tmp/C/76340497696140b7b91858c99dc3f992 is 50, key is test_row_0/C:col10/1732246953031/Put/seqid=0 2024-11-22T03:42:34,196 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742334_1510 (size=12001) 2024-11-22T03:42:34,196 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=117 (bloomFilter=true), to=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/.tmp/C/76340497696140b7b91858c99dc3f992 2024-11-22T03:42:34,200 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/.tmp/A/f3a5993a5b7a4d7baccc7a9b2f3d5aaf as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/A/f3a5993a5b7a4d7baccc7a9b2f3d5aaf 2024-11-22T03:42:34,203 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/A/f3a5993a5b7a4d7baccc7a9b2f3d5aaf, entries=200, sequenceid=117, filesize=38.6 K 2024-11-22T03:42:34,204 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/.tmp/B/95ac750810af44f6b683b180c5c55c4d as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/B/95ac750810af44f6b683b180c5c55c4d 2024-11-22T03:42:34,208 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/B/95ac750810af44f6b683b180c5c55c4d, entries=150, sequenceid=117, filesize=11.7 K 2024-11-22T03:42:34,209 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/.tmp/C/76340497696140b7b91858c99dc3f992 as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/C/76340497696140b7b91858c99dc3f992 2024-11-22T03:42:34,211 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/C/76340497696140b7b91858c99dc3f992, entries=150, sequenceid=117, filesize=11.7 K 2024-11-22T03:42:34,212 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for 8422287be02eb151a34ae7fe752afee0 in 861ms, sequenceid=117, compaction requested=true 2024-11-22T03:42:34,212 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8422287be02eb151a34ae7fe752afee0: 2024-11-22T03:42:34,212 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8422287be02eb151a34ae7fe752afee0:A, priority=-2147483648, current under compaction store size is 1 2024-11-22T03:42:34,212 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T03:42:34,212 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-22T03:42:34,212 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8422287be02eb151a34ae7fe752afee0:B, priority=-2147483648, current under compaction store size is 2 2024-11-22T03:42:34,212 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T03:42:34,212 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8422287be02eb151a34ae7fe752afee0:C, priority=-2147483648, current under compaction store size is 3 2024-11-22T03:42:34,212 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-22T03:42:34,212 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T03:42:34,214 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-22T03:42:34,214 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 141111 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-22T03:42:34,214 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.HStore(1541): 8422287be02eb151a34ae7fe752afee0/A is initiating minor compaction (all files) 2024-11-22T03:42:34,214 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HStore(1541): 8422287be02eb151a34ae7fe752afee0/B is initiating minor compaction (all files) 2024-11-22T03:42:34,214 INFO [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8422287be02eb151a34ae7fe752afee0/A in TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0. 2024-11-22T03:42:34,214 INFO [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8422287be02eb151a34ae7fe752afee0/B in TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0. 2024-11-22T03:42:34,214 INFO [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/B/d85467796c174b3b9d6176079597e3cf, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/B/7c04c26321b94ba8b67a3c48d13e214e, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/B/276fa3c198bf43f0af4cdee49a168057, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/B/95ac750810af44f6b683b180c5c55c4d] into tmpdir=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/.tmp, totalSize=47.0 K 2024-11-22T03:42:34,214 INFO [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/A/84c77362f3a8479ba4a6997ea26478a5, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/A/f58c125a588e45e08433dff8062a916c, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/A/08d77b7752a94b26971efbb82978da3b, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/A/f3a5993a5b7a4d7baccc7a9b2f3d5aaf] into tmpdir=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/.tmp, totalSize=137.8 K 2024-11-22T03:42:34,214 INFO [RS:0;c85114ed5096:42545-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0. 2024-11-22T03:42:34,214 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0. files: [hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/A/84c77362f3a8479ba4a6997ea26478a5, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/A/f58c125a588e45e08433dff8062a916c, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/A/08d77b7752a94b26971efbb82978da3b, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/A/f3a5993a5b7a4d7baccc7a9b2f3d5aaf] 2024-11-22T03:42:34,214 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] compactions.Compactor(225): Compacting d85467796c174b3b9d6176079597e3cf, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1732246948714 2024-11-22T03:42:34,215 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] compactions.Compactor(225): Compacting 84c77362f3a8479ba4a6997ea26478a5, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1732246948714 2024-11-22T03:42:34,215 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] compactions.Compactor(225): Compacting 7c04c26321b94ba8b67a3c48d13e214e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=80, earliestPutTs=1732246948881 2024-11-22T03:42:34,215 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] compactions.Compactor(225): Compacting 276fa3c198bf43f0af4cdee49a168057, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=91, earliestPutTs=1732246950788 2024-11-22T03:42:34,215 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] compactions.Compactor(225): Compacting f58c125a588e45e08433dff8062a916c, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=80, earliestPutTs=1732246948881 2024-11-22T03:42:34,215 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] compactions.Compactor(225): Compacting 95ac750810af44f6b683b180c5c55c4d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1732246953011 2024-11-22T03:42:34,215 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] compactions.Compactor(225): Compacting 08d77b7752a94b26971efbb82978da3b, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=91, earliestPutTs=1732246950788 2024-11-22T03:42:34,215 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] compactions.Compactor(225): Compacting f3a5993a5b7a4d7baccc7a9b2f3d5aaf, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1732246953011 2024-11-22T03:42:34,239 INFO [RS:0;c85114ed5096:42545-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8422287be02eb151a34ae7fe752afee0#B#compaction#435 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T03:42:34,239 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/.tmp/B/651411e40abe4a46987cf338155c610b is 50, key is test_row_0/B:col10/1732246953031/Put/seqid=0 2024-11-22T03:42:34,241 INFO [RS:0;c85114ed5096:42545-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=8422287be02eb151a34ae7fe752afee0] 2024-11-22T03:42:34,245 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411228df22e8ce0e748df8666b80bb3696c90_8422287be02eb151a34ae7fe752afee0 store=[table=TestAcidGuarantees family=A region=8422287be02eb151a34ae7fe752afee0] 2024-11-22T03:42:34,247 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411228df22e8ce0e748df8666b80bb3696c90_8422287be02eb151a34ae7fe752afee0, store=[table=TestAcidGuarantees family=A region=8422287be02eb151a34ae7fe752afee0] 2024-11-22T03:42:34,248 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411228df22e8ce0e748df8666b80bb3696c90_8422287be02eb151a34ae7fe752afee0 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=8422287be02eb151a34ae7fe752afee0] 2024-11-22T03:42:34,288 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742335_1511 (size=12241) 2024-11-22T03:42:34,303 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742336_1512 (size=4469) 2024-11-22T03:42:34,306 INFO [RS:0;c85114ed5096:42545-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8422287be02eb151a34ae7fe752afee0#A#compaction#436 average throughput is 0.38 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T03:42:34,306 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/.tmp/A/991973affb45468ab95b18ed8152698d is 175, key is test_row_0/A:col10/1732246953031/Put/seqid=0 2024-11-22T03:42:34,319 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742337_1513 (size=31195) 2024-11-22T03:42:34,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(8855): Flush requested on 8422287be02eb151a34ae7fe752afee0 2024-11-22T03:42:34,520 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-22T03:42:34,520 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8422287be02eb151a34ae7fe752afee0 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-22T03:42:34,537 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241122b1c73c919e654418adb41947c1fedffd_8422287be02eb151a34ae7fe752afee0 is 50, key is test_row_0/A:col10/1732246953376/Put/seqid=0 2024-11-22T03:42:34,540 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742338_1514 (size=12154) 2024-11-22T03:42:34,633 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:42:34,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37380 deadline: 1732247014628, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 2024-11-22T03:42:34,633 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:34,633 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:34,633 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:42:34,691 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/.tmp/B/651411e40abe4a46987cf338155c610b as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/B/651411e40abe4a46987cf338155c610b 2024-11-22T03:42:34,695 INFO [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 8422287be02eb151a34ae7fe752afee0/B of 8422287be02eb151a34ae7fe752afee0 into 651411e40abe4a46987cf338155c610b(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T03:42:34,695 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8422287be02eb151a34ae7fe752afee0: 2024-11-22T03:42:34,695 INFO [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., storeName=8422287be02eb151a34ae7fe752afee0/B, priority=12, startTime=1732246954212; duration=0sec 2024-11-22T03:42:34,695 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T03:42:34,695 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8422287be02eb151a34ae7fe752afee0:B 2024-11-22T03:42:34,695 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-22T03:42:34,696 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-22T03:42:34,696 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HStore(1541): 8422287be02eb151a34ae7fe752afee0/C is initiating minor compaction (all files) 2024-11-22T03:42:34,696 INFO [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8422287be02eb151a34ae7fe752afee0/C in TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0. 2024-11-22T03:42:34,696 INFO [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/C/6a403ba20043415fa48677d75ee90ed6, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/C/f9056aac4dc740059dd3efb37f86d910, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/C/f370abdfa2174d93bdfd2318f0c74cb7, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/C/76340497696140b7b91858c99dc3f992] into tmpdir=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/.tmp, totalSize=47.0 K 2024-11-22T03:42:34,696 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] compactions.Compactor(225): Compacting 6a403ba20043415fa48677d75ee90ed6, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1732246948714 2024-11-22T03:42:34,696 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] compactions.Compactor(225): Compacting f9056aac4dc740059dd3efb37f86d910, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=80, earliestPutTs=1732246948881 2024-11-22T03:42:34,697 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] compactions.Compactor(225): Compacting f370abdfa2174d93bdfd2318f0c74cb7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=91, earliestPutTs=1732246950788 2024-11-22T03:42:34,697 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] compactions.Compactor(225): Compacting 76340497696140b7b91858c99dc3f992, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1732246953011 2024-11-22T03:42:34,704 INFO [RS:0;c85114ed5096:42545-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8422287be02eb151a34ae7fe752afee0#C#compaction#438 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T03:42:34,705 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/.tmp/C/c23efac2b8274234a93b10b14a7cd17c is 50, key is test_row_0/C:col10/1732246953031/Put/seqid=0 2024-11-22T03:42:34,721 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742339_1515 (size=12241) 2024-11-22T03:42:34,726 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/.tmp/C/c23efac2b8274234a93b10b14a7cd17c as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/C/c23efac2b8274234a93b10b14a7cd17c 2024-11-22T03:42:34,726 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/.tmp/A/991973affb45468ab95b18ed8152698d as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/A/991973affb45468ab95b18ed8152698d 2024-11-22T03:42:34,729 INFO [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 8422287be02eb151a34ae7fe752afee0/A of 8422287be02eb151a34ae7fe752afee0 into 991973affb45468ab95b18ed8152698d(size=30.5 K), total size for store is 30.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T03:42:34,729 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8422287be02eb151a34ae7fe752afee0: 2024-11-22T03:42:34,730 INFO [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., storeName=8422287be02eb151a34ae7fe752afee0/A, priority=12, startTime=1732246954212; duration=0sec 2024-11-22T03:42:34,730 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T03:42:34,730 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8422287be02eb151a34ae7fe752afee0:A 2024-11-22T03:42:34,731 INFO [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 8422287be02eb151a34ae7fe752afee0/C of 8422287be02eb151a34ae7fe752afee0 into c23efac2b8274234a93b10b14a7cd17c(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T03:42:34,731 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8422287be02eb151a34ae7fe752afee0: 2024-11-22T03:42:34,731 INFO [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., storeName=8422287be02eb151a34ae7fe752afee0/C, priority=12, startTime=1732246954212; duration=0sec 2024-11-22T03:42:34,731 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T03:42:34,731 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8422287be02eb151a34ae7fe752afee0:C 2024-11-22T03:42:34,741 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:42:34,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37380 deadline: 1732247014739, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 2024-11-22T03:42:34,742 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:34,742 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:34,742 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:42:34,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-22T03:42:34,849 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-22T03:42:34,849 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42627 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-22T03:42:34,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42627 {}] procedure2.ProcedureExecutor(1139): Stored pid=154, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees 2024-11-22T03:42:34,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-22T03:42:34,851 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=154, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-22T03:42:34,851 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=154, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-22T03:42:34,851 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=155, ppid=154, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-22T03:42:34,941 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:42:34,943 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241122b1c73c919e654418adb41947c1fedffd_8422287be02eb151a34ae7fe752afee0 to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241122b1c73c919e654418adb41947c1fedffd_8422287be02eb151a34ae7fe752afee0 2024-11-22T03:42:34,944 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/.tmp/A/1a92649cae8d4751903576d32badb85b, store: [table=TestAcidGuarantees family=A region=8422287be02eb151a34ae7fe752afee0] 2024-11-22T03:42:34,945 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/.tmp/A/1a92649cae8d4751903576d32badb85b is 175, key is test_row_0/A:col10/1732246953376/Put/seqid=0 2024-11-22T03:42:34,950 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:42:34,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37380 deadline: 1732247014948, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 2024-11-22T03:42:34,951 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:34,951 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:34,951 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:42:34,956 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742340_1516 (size=30955) 2024-11-22T03:42:34,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-22T03:42:35,003 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42545 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-22T03:42:35,003 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0. 2024-11-22T03:42:35,003 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0. as already flushing 2024-11-22T03:42:35,003 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0. 2024-11-22T03:42:35,003 ERROR [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 8422287be02eb151a34ae7fe752afee0, NAME => 'TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:42:35,003 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 8422287be02eb151a34ae7fe752afee0, NAME => 'TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:42:35,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42627 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8422287be02eb151a34ae7fe752afee0, NAME => 'TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8422287be02eb151a34ae7fe752afee0, NAME => 'TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:42:35,155 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42545 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-22T03:42:35,155 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0. 2024-11-22T03:42:35,156 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0. as already flushing 2024-11-22T03:42:35,156 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0. 2024-11-22T03:42:35,156 ERROR [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 8422287be02eb151a34ae7fe752afee0, NAME => 'TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:42:35,156 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 8422287be02eb151a34ae7fe752afee0, NAME => 'TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:42:35,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42627 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8422287be02eb151a34ae7fe752afee0, NAME => 'TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8422287be02eb151a34ae7fe752afee0, NAME => 'TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:42:35,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-22T03:42:35,259 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:42:35,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37380 deadline: 1732247015258, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 2024-11-22T03:42:35,260 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:35,260 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:35,260 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:42:35,307 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42545 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-22T03:42:35,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0. 2024-11-22T03:42:35,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0. as already flushing 2024-11-22T03:42:35,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0. 2024-11-22T03:42:35,308 ERROR [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 8422287be02eb151a34ae7fe752afee0, NAME => 'TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:42:35,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 8422287be02eb151a34ae7fe752afee0, NAME => 'TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:42:35,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42627 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8422287be02eb151a34ae7fe752afee0, NAME => 'TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8422287be02eb151a34ae7fe752afee0, NAME => 'TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:42:35,357 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=128, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/.tmp/A/1a92649cae8d4751903576d32badb85b 2024-11-22T03:42:35,362 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/.tmp/B/477f1f9796664cde8f850858df092ebd is 50, key is test_row_0/B:col10/1732246953376/Put/seqid=0 2024-11-22T03:42:35,378 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742341_1517 (size=12001) 2024-11-22T03:42:35,460 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42545 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-22T03:42:35,460 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0. 2024-11-22T03:42:35,460 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0. as already flushing 2024-11-22T03:42:35,460 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0. 2024-11-22T03:42:35,460 ERROR [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 8422287be02eb151a34ae7fe752afee0, NAME => 'TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:42:35,460 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 8422287be02eb151a34ae7fe752afee0, NAME => 'TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:42:35,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42627 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8422287be02eb151a34ae7fe752afee0, NAME => 'TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8422287be02eb151a34ae7fe752afee0, NAME => 'TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:42:35,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-22T03:42:35,612 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42545 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-22T03:42:35,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0. 2024-11-22T03:42:35,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0. as already flushing 2024-11-22T03:42:35,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0. 2024-11-22T03:42:35,613 ERROR [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 8422287be02eb151a34ae7fe752afee0, NAME => 'TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:42:35,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 8422287be02eb151a34ae7fe752afee0, NAME => 'TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:42:35,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42627 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8422287be02eb151a34ae7fe752afee0, NAME => 'TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8422287be02eb151a34ae7fe752afee0, NAME => 'TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:42:35,765 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42545 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-22T03:42:35,765 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0. 2024-11-22T03:42:35,765 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0. as already flushing 2024-11-22T03:42:35,765 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0. 2024-11-22T03:42:35,765 ERROR [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 8422287be02eb151a34ae7fe752afee0, NAME => 'TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:42:35,765 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 8422287be02eb151a34ae7fe752afee0, NAME => 'TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:42:35,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42627 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8422287be02eb151a34ae7fe752afee0, NAME => 'TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8422287be02eb151a34ae7fe752afee0, NAME => 'TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:42:35,771 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:42:35,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37380 deadline: 1732247015769, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 2024-11-22T03:42:35,771 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:35,771 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:35,771 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:42:35,779 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=128 (bloomFilter=true), to=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/.tmp/B/477f1f9796664cde8f850858df092ebd 2024-11-22T03:42:35,784 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/.tmp/C/fc6202d01a1d4f38be8f5a1a5a04eda8 is 50, key is test_row_0/C:col10/1732246953376/Put/seqid=0 2024-11-22T03:42:35,787 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742342_1518 (size=12001) 2024-11-22T03:42:35,918 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42545 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-22T03:42:35,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0. 2024-11-22T03:42:35,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0. as already flushing 2024-11-22T03:42:35,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0. 2024-11-22T03:42:35,918 ERROR [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 8422287be02eb151a34ae7fe752afee0, NAME => 'TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:42:35,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 8422287be02eb151a34ae7fe752afee0, NAME => 'TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:42:35,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42627 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8422287be02eb151a34ae7fe752afee0, NAME => 'TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8422287be02eb151a34ae7fe752afee0, NAME => 'TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:42:35,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-22T03:42:36,070 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42545 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-22T03:42:36,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0. 2024-11-22T03:42:36,071 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0. as already flushing 2024-11-22T03:42:36,071 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0. 2024-11-22T03:42:36,071 ERROR [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 8422287be02eb151a34ae7fe752afee0, NAME => 'TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:42:36,071 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 8422287be02eb151a34ae7fe752afee0, NAME => 'TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:42:36,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42627 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8422287be02eb151a34ae7fe752afee0, NAME => 'TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8422287be02eb151a34ae7fe752afee0, NAME => 'TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:42:36,188 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-22T03:42:36,188 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-11-22T03:42:36,188 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=128 (bloomFilter=true), to=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/.tmp/C/fc6202d01a1d4f38be8f5a1a5a04eda8 2024-11-22T03:42:36,191 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/.tmp/A/1a92649cae8d4751903576d32badb85b as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/A/1a92649cae8d4751903576d32badb85b 2024-11-22T03:42:36,194 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/A/1a92649cae8d4751903576d32badb85b, entries=150, sequenceid=128, filesize=30.2 K 2024-11-22T03:42:36,195 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/.tmp/B/477f1f9796664cde8f850858df092ebd as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/B/477f1f9796664cde8f850858df092ebd 2024-11-22T03:42:36,197 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/B/477f1f9796664cde8f850858df092ebd, entries=150, sequenceid=128, filesize=11.7 K 2024-11-22T03:42:36,198 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/.tmp/C/fc6202d01a1d4f38be8f5a1a5a04eda8 as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/C/fc6202d01a1d4f38be8f5a1a5a04eda8 2024-11-22T03:42:36,200 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/C/fc6202d01a1d4f38be8f5a1a5a04eda8, entries=150, sequenceid=128, filesize=11.7 K 2024-11-22T03:42:36,201 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 8422287be02eb151a34ae7fe752afee0 in 1681ms, sequenceid=128, compaction requested=false 2024-11-22T03:42:36,201 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8422287be02eb151a34ae7fe752afee0: 2024-11-22T03:42:36,223 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42545 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-22T03:42:36,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0. 2024-11-22T03:42:36,223 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2902): Flushing 8422287be02eb151a34ae7fe752afee0 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-22T03:42:36,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411226e41f0edd0a04841b36d952b4689f447_8422287be02eb151a34ae7fe752afee0 is 50, key is test_row_0/A:col10/1732246954626/Put/seqid=0 2024-11-22T03:42:36,234 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742343_1519 (size=12304) 2024-11-22T03:42:36,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:42:36,238 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411226e41f0edd0a04841b36d952b4689f447_8422287be02eb151a34ae7fe752afee0 to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411226e41f0edd0a04841b36d952b4689f447_8422287be02eb151a34ae7fe752afee0 2024-11-22T03:42:36,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/.tmp/A/c3e84477a4504fc4bf2e7dba0c087cc5, store: [table=TestAcidGuarantees family=A region=8422287be02eb151a34ae7fe752afee0] 2024-11-22T03:42:36,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/.tmp/A/c3e84477a4504fc4bf2e7dba0c087cc5 is 175, key is test_row_0/A:col10/1732246954626/Put/seqid=0 2024-11-22T03:42:36,243 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742344_1520 (size=31105) 2024-11-22T03:42:36,245 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=156, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/.tmp/A/c3e84477a4504fc4bf2e7dba0c087cc5 2024-11-22T03:42:36,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/.tmp/B/bf28dcbf9bea40f1a60b37230368093e is 50, key is test_row_0/B:col10/1732246954626/Put/seqid=0 2024-11-22T03:42:36,266 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742345_1521 (size=12151) 2024-11-22T03:42:36,667 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=156 (bloomFilter=true), to=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/.tmp/B/bf28dcbf9bea40f1a60b37230368093e 2024-11-22T03:42:36,673 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/.tmp/C/5b22cb77a3fd48f29c3e34d53305de40 is 50, key is test_row_0/C:col10/1732246954626/Put/seqid=0 2024-11-22T03:42:36,695 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742346_1522 (size=12151) 2024-11-22T03:42:36,790 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0. as already flushing 2024-11-22T03:42:36,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(8855): Flush requested on 8422287be02eb151a34ae7fe752afee0 2024-11-22T03:42:36,817 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:42:36,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37380 deadline: 1732247016814, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 2024-11-22T03:42:36,817 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:36,817 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:36,817 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:42:36,931 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:42:36,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37380 deadline: 1732247016928, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 2024-11-22T03:42:36,932 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:36,932 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:36,932 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:42:36,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-22T03:42:37,061 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:42:37,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37400 deadline: 1732247017059, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 2024-11-22T03:42:37,061 WARN [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRpcRetryingCaller(177): Call to c85114ed5096,42545,1732246786482 for 'test_row_1' in 8422287be02eb151a34ae7fe752afee0 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8171 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-22T03:42:37,062 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:37,062 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:37,062 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:42:37,089 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:42:37,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37416 deadline: 1732247017088, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 2024-11-22T03:42:37,090 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to c85114ed5096,42545,1732246786482 for 'test_row_0' in 8422287be02eb151a34ae7fe752afee0 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8205 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-22T03:42:37,090 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:37,090 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:37,090 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:42:37,095 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=156 (bloomFilter=true), to=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/.tmp/C/5b22cb77a3fd48f29c3e34d53305de40 2024-11-22T03:42:37,098 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/.tmp/A/c3e84477a4504fc4bf2e7dba0c087cc5 as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/A/c3e84477a4504fc4bf2e7dba0c087cc5 2024-11-22T03:42:37,102 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/A/c3e84477a4504fc4bf2e7dba0c087cc5, entries=150, sequenceid=156, filesize=30.4 K 2024-11-22T03:42:37,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/.tmp/B/bf28dcbf9bea40f1a60b37230368093e as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/B/bf28dcbf9bea40f1a60b37230368093e 2024-11-22T03:42:37,106 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/B/bf28dcbf9bea40f1a60b37230368093e, entries=150, sequenceid=156, filesize=11.9 K 2024-11-22T03:42:37,106 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:42:37,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37384 deadline: 1732247017105, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 2024-11-22T03:42:37,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/.tmp/C/5b22cb77a3fd48f29c3e34d53305de40 as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/C/5b22cb77a3fd48f29c3e34d53305de40 2024-11-22T03:42:37,107 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to c85114ed5096,42545,1732246786482 for 'test_row_1' in 8422287be02eb151a34ae7fe752afee0 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8221 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-22T03:42:37,107 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:37,107 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:37,107 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:42:37,110 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/C/5b22cb77a3fd48f29c3e34d53305de40, entries=150, sequenceid=156, filesize=11.9 K 2024-11-22T03:42:37,111 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 8422287be02eb151a34ae7fe752afee0 in 888ms, sequenceid=156, compaction requested=true 2024-11-22T03:42:37,111 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2603): Flush status journal for 8422287be02eb151a34ae7fe752afee0: 2024-11-22T03:42:37,111 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0. 2024-11-22T03:42:37,111 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=155 2024-11-22T03:42:37,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42627 {}] master.HMaster(4169): Remote procedure done, pid=155 2024-11-22T03:42:37,113 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-22T03:42:37,113 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8422287be02eb151a34ae7fe752afee0 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-22T03:42:37,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(8855): Flush requested on 8422287be02eb151a34ae7fe752afee0 2024-11-22T03:42:37,114 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=155, resume processing ppid=154 2024-11-22T03:42:37,114 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=155, ppid=154, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2610 sec 2024-11-22T03:42:37,115 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=154, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees in 2.2650 sec 2024-11-22T03:42:37,119 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241122435db259e16b4ddd868a08fad804a3fa_8422287be02eb151a34ae7fe752afee0 is 50, key is test_row_0/A:col10/1732246956813/Put/seqid=0 2024-11-22T03:42:37,122 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742347_1523 (size=14794) 2024-11-22T03:42:37,176 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:42:37,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37382 deadline: 1732247017173, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 2024-11-22T03:42:37,177 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:37,177 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:37,177 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:42:37,179 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:42:37,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37380 deadline: 1732247017176, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 2024-11-22T03:42:37,180 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:37,180 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:37,180 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:42:37,282 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:42:37,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37382 deadline: 1732247017279, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 2024-11-22T03:42:37,283 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:37,283 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:37,283 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:42:37,290 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:42:37,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37380 deadline: 1732247017288, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 2024-11-22T03:42:37,290 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:37,291 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:37,291 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:42:37,491 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:42:37,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37382 deadline: 1732247017489, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 2024-11-22T03:42:37,492 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:37,492 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:37,492 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:42:37,500 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:42:37,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37380 deadline: 1732247017499, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 2024-11-22T03:42:37,501 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:37,501 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:37,501 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:42:37,523 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:42:37,525 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241122435db259e16b4ddd868a08fad804a3fa_8422287be02eb151a34ae7fe752afee0 to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241122435db259e16b4ddd868a08fad804a3fa_8422287be02eb151a34ae7fe752afee0 2024-11-22T03:42:37,526 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/.tmp/A/13f7cb4e7374438d8bd0d585c0c38610, store: [table=TestAcidGuarantees family=A region=8422287be02eb151a34ae7fe752afee0] 2024-11-22T03:42:37,527 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/.tmp/A/13f7cb4e7374438d8bd0d585c0c38610 is 175, key is test_row_0/A:col10/1732246956813/Put/seqid=0 2024-11-22T03:42:37,530 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742348_1524 (size=39749) 2024-11-22T03:42:37,530 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=168, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/.tmp/A/13f7cb4e7374438d8bd0d585c0c38610 2024-11-22T03:42:37,535 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/.tmp/B/9a2d7fe083cf4333a06fb1455ffde479 is 50, key is test_row_0/B:col10/1732246956813/Put/seqid=0 2024-11-22T03:42:37,538 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742349_1525 (size=12151) 2024-11-22T03:42:37,800 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:42:37,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37382 deadline: 1732247017798, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 2024-11-22T03:42:37,800 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:37,800 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:37,800 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:42:37,810 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:42:37,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37380 deadline: 1732247017808, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 2024-11-22T03:42:37,810 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:37,810 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:37,810 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:42:37,939 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=168 (bloomFilter=true), to=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/.tmp/B/9a2d7fe083cf4333a06fb1455ffde479 2024-11-22T03:42:37,944 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/.tmp/C/0e93261a48454456868712b724585666 is 50, key is test_row_0/C:col10/1732246956813/Put/seqid=0 2024-11-22T03:42:37,954 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742350_1526 (size=12151) 2024-11-22T03:42:38,312 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:42:38,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37382 deadline: 1732247018309, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 2024-11-22T03:42:38,313 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:38,313 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:38,313 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:42:38,321 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:42:38,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37380 deadline: 1732247018318, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 2024-11-22T03:42:38,321 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:38,321 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:38,321 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:42:38,354 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=168 (bloomFilter=true), to=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/.tmp/C/0e93261a48454456868712b724585666 2024-11-22T03:42:38,357 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/.tmp/A/13f7cb4e7374438d8bd0d585c0c38610 as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/A/13f7cb4e7374438d8bd0d585c0c38610 2024-11-22T03:42:38,360 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/A/13f7cb4e7374438d8bd0d585c0c38610, entries=200, sequenceid=168, filesize=38.8 K 2024-11-22T03:42:38,361 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/.tmp/B/9a2d7fe083cf4333a06fb1455ffde479 as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/B/9a2d7fe083cf4333a06fb1455ffde479 2024-11-22T03:42:38,363 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/B/9a2d7fe083cf4333a06fb1455ffde479, entries=150, sequenceid=168, filesize=11.9 K 2024-11-22T03:42:38,363 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/.tmp/C/0e93261a48454456868712b724585666 as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/C/0e93261a48454456868712b724585666 2024-11-22T03:42:38,366 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/C/0e93261a48454456868712b724585666, entries=150, sequenceid=168, filesize=11.9 K 2024-11-22T03:42:38,367 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 8422287be02eb151a34ae7fe752afee0 in 1254ms, sequenceid=168, compaction requested=true 2024-11-22T03:42:38,367 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8422287be02eb151a34ae7fe752afee0: 2024-11-22T03:42:38,367 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8422287be02eb151a34ae7fe752afee0:A, priority=-2147483648, current under compaction store size is 1 2024-11-22T03:42:38,367 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T03:42:38,367 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-22T03:42:38,367 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8422287be02eb151a34ae7fe752afee0:B, priority=-2147483648, current under compaction store size is 2 2024-11-22T03:42:38,367 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T03:42:38,367 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-22T03:42:38,367 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8422287be02eb151a34ae7fe752afee0:C, priority=-2147483648, current under compaction store size is 3 2024-11-22T03:42:38,367 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T03:42:38,368 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48544 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-22T03:42:38,368 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 133004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-22T03:42:38,368 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HStore(1541): 8422287be02eb151a34ae7fe752afee0/B is initiating minor compaction (all files) 2024-11-22T03:42:38,368 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.HStore(1541): 8422287be02eb151a34ae7fe752afee0/A is initiating minor compaction (all files) 2024-11-22T03:42:38,368 INFO [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8422287be02eb151a34ae7fe752afee0/B in TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0. 2024-11-22T03:42:38,368 INFO [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/B/651411e40abe4a46987cf338155c610b, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/B/477f1f9796664cde8f850858df092ebd, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/B/bf28dcbf9bea40f1a60b37230368093e, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/B/9a2d7fe083cf4333a06fb1455ffde479] into tmpdir=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/.tmp, totalSize=47.4 K 2024-11-22T03:42:38,368 INFO [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8422287be02eb151a34ae7fe752afee0/A in TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0. 2024-11-22T03:42:38,368 INFO [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/A/991973affb45468ab95b18ed8152698d, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/A/1a92649cae8d4751903576d32badb85b, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/A/c3e84477a4504fc4bf2e7dba0c087cc5, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/A/13f7cb4e7374438d8bd0d585c0c38610] into tmpdir=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/.tmp, totalSize=129.9 K 2024-11-22T03:42:38,368 INFO [RS:0;c85114ed5096:42545-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0. 2024-11-22T03:42:38,369 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0. files: [hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/A/991973affb45468ab95b18ed8152698d, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/A/1a92649cae8d4751903576d32badb85b, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/A/c3e84477a4504fc4bf2e7dba0c087cc5, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/A/13f7cb4e7374438d8bd0d585c0c38610] 2024-11-22T03:42:38,369 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] compactions.Compactor(225): Compacting 651411e40abe4a46987cf338155c610b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1732246953011 2024-11-22T03:42:38,369 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] compactions.Compactor(225): Compacting 991973affb45468ab95b18ed8152698d, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1732246953011 2024-11-22T03:42:38,369 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] compactions.Compactor(225): Compacting 477f1f9796664cde8f850858df092ebd, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=128, earliestPutTs=1732246953376 2024-11-22T03:42:38,369 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1a92649cae8d4751903576d32badb85b, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=128, earliestPutTs=1732246953376 2024-11-22T03:42:38,370 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] compactions.Compactor(225): Compacting c3e84477a4504fc4bf2e7dba0c087cc5, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1732246954598 2024-11-22T03:42:38,370 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] compactions.Compactor(225): Compacting bf28dcbf9bea40f1a60b37230368093e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1732246954598 2024-11-22T03:42:38,370 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] compactions.Compactor(225): Compacting 9a2d7fe083cf4333a06fb1455ffde479, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=168, earliestPutTs=1732246956806 2024-11-22T03:42:38,370 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] compactions.Compactor(225): Compacting 13f7cb4e7374438d8bd0d585c0c38610, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=168, earliestPutTs=1732246956806 2024-11-22T03:42:38,378 INFO [RS:0;c85114ed5096:42545-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=8422287be02eb151a34ae7fe752afee0] 2024-11-22T03:42:38,381 INFO [RS:0;c85114ed5096:42545-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8422287be02eb151a34ae7fe752afee0#B#compaction#448 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-22T03:42:38,381 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/.tmp/B/05ad277a790e445eb8957f5e67a78ee7 is 50, key is test_row_0/B:col10/1732246956813/Put/seqid=0 2024-11-22T03:42:38,385 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411229184077f666e42869f8427dfb8c1b2eb_8422287be02eb151a34ae7fe752afee0 store=[table=TestAcidGuarantees family=A region=8422287be02eb151a34ae7fe752afee0] 2024-11-22T03:42:38,388 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411229184077f666e42869f8427dfb8c1b2eb_8422287be02eb151a34ae7fe752afee0, store=[table=TestAcidGuarantees family=A region=8422287be02eb151a34ae7fe752afee0] 2024-11-22T03:42:38,388 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411229184077f666e42869f8427dfb8c1b2eb_8422287be02eb151a34ae7fe752afee0 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=8422287be02eb151a34ae7fe752afee0] 2024-11-22T03:42:38,412 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742352_1528 (size=4469) 2024-11-22T03:42:38,412 INFO [RS:0;c85114ed5096:42545-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8422287be02eb151a34ae7fe752afee0#A#compaction#447 average throughput is 0.72 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T03:42:38,413 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/.tmp/A/d0625ea7b4304705aabef95e0c1ecfe5 is 175, key is test_row_0/A:col10/1732246956813/Put/seqid=0 2024-11-22T03:42:38,414 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742351_1527 (size=12527) 2024-11-22T03:42:38,431 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742353_1529 (size=31481) 2024-11-22T03:42:38,431 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/.tmp/B/05ad277a790e445eb8957f5e67a78ee7 as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/B/05ad277a790e445eb8957f5e67a78ee7 2024-11-22T03:42:38,436 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/.tmp/A/d0625ea7b4304705aabef95e0c1ecfe5 as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/A/d0625ea7b4304705aabef95e0c1ecfe5 2024-11-22T03:42:38,437 INFO [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 8422287be02eb151a34ae7fe752afee0/B of 8422287be02eb151a34ae7fe752afee0 into 05ad277a790e445eb8957f5e67a78ee7(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T03:42:38,437 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8422287be02eb151a34ae7fe752afee0: 2024-11-22T03:42:38,437 INFO [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., storeName=8422287be02eb151a34ae7fe752afee0/B, priority=12, startTime=1732246958367; duration=0sec 2024-11-22T03:42:38,437 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T03:42:38,438 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8422287be02eb151a34ae7fe752afee0:B 2024-11-22T03:42:38,438 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-22T03:42:38,440 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48544 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-22T03:42:38,440 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HStore(1541): 8422287be02eb151a34ae7fe752afee0/C is initiating minor compaction (all files) 2024-11-22T03:42:38,440 INFO [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8422287be02eb151a34ae7fe752afee0/C in TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0. 2024-11-22T03:42:38,440 INFO [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/C/c23efac2b8274234a93b10b14a7cd17c, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/C/fc6202d01a1d4f38be8f5a1a5a04eda8, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/C/5b22cb77a3fd48f29c3e34d53305de40, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/C/0e93261a48454456868712b724585666] into tmpdir=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/.tmp, totalSize=47.4 K 2024-11-22T03:42:38,441 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] compactions.Compactor(225): Compacting c23efac2b8274234a93b10b14a7cd17c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1732246953011 2024-11-22T03:42:38,441 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] compactions.Compactor(225): Compacting fc6202d01a1d4f38be8f5a1a5a04eda8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=128, earliestPutTs=1732246953376 2024-11-22T03:42:38,442 INFO [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 8422287be02eb151a34ae7fe752afee0/A of 8422287be02eb151a34ae7fe752afee0 into d0625ea7b4304705aabef95e0c1ecfe5(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T03:42:38,442 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8422287be02eb151a34ae7fe752afee0: 2024-11-22T03:42:38,442 INFO [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., storeName=8422287be02eb151a34ae7fe752afee0/A, priority=12, startTime=1732246958367; duration=0sec 2024-11-22T03:42:38,442 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T03:42:38,442 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8422287be02eb151a34ae7fe752afee0:A 2024-11-22T03:42:38,443 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] compactions.Compactor(225): Compacting 5b22cb77a3fd48f29c3e34d53305de40, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1732246954598 2024-11-22T03:42:38,443 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] compactions.Compactor(225): Compacting 0e93261a48454456868712b724585666, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=168, earliestPutTs=1732246956806 2024-11-22T03:42:38,451 INFO [RS:0;c85114ed5096:42545-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8422287be02eb151a34ae7fe752afee0#C#compaction#449 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T03:42:38,451 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/.tmp/C/848f862c06e847e683a59ec7e0c11d81 is 50, key is test_row_0/C:col10/1732246956813/Put/seqid=0 2024-11-22T03:42:38,487 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742354_1530 (size=12527) 2024-11-22T03:42:38,494 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/.tmp/C/848f862c06e847e683a59ec7e0c11d81 as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/C/848f862c06e847e683a59ec7e0c11d81 2024-11-22T03:42:38,502 INFO [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 8422287be02eb151a34ae7fe752afee0/C of 8422287be02eb151a34ae7fe752afee0 into 848f862c06e847e683a59ec7e0c11d81(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T03:42:38,502 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8422287be02eb151a34ae7fe752afee0: 2024-11-22T03:42:38,502 INFO [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., storeName=8422287be02eb151a34ae7fe752afee0/C, priority=12, startTime=1732246958367; duration=0sec 2024-11-22T03:42:38,503 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T03:42:38,503 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8422287be02eb151a34ae7fe752afee0:C 2024-11-22T03:42:39,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-22T03:42:39,018 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-22T03:42:39,019 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42627 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-22T03:42:39,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42627 {}] procedure2.ProcedureExecutor(1139): Stored pid=156, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees 2024-11-22T03:42:39,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-22T03:42:39,020 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=156, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-22T03:42:39,021 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=156, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-22T03:42:39,021 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=157, ppid=156, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-22T03:42:39,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-22T03:42:39,173 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42545 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-22T03:42:39,173 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0. 2024-11-22T03:42:39,173 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2902): Flushing 8422287be02eb151a34ae7fe752afee0 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-22T03:42:39,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112243bdd4c1a8344ad7a9fe7f80ca24ead1_8422287be02eb151a34ae7fe752afee0 is 50, key is test_row_0/A:col10/1732246957167/Put/seqid=0 2024-11-22T03:42:39,181 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742355_1531 (size=12304) 2024-11-22T03:42:39,330 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0. as already flushing 2024-11-22T03:42:39,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(8855): Flush requested on 8422287be02eb151a34ae7fe752afee0 2024-11-22T03:42:39,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-22T03:42:39,346 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:42:39,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37380 deadline: 1732247019342, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 2024-11-22T03:42:39,346 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:39,346 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:39,346 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:42:39,348 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:42:39,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37382 deadline: 1732247019345, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 2024-11-22T03:42:39,348 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:39,348 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:39,348 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:42:39,452 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:42:39,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37380 deadline: 1732247019449, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 2024-11-22T03:42:39,452 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:39,452 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:39,452 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:42:39,460 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:42:39,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37382 deadline: 1732247019459, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 2024-11-22T03:42:39,460 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:39,461 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:39,461 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:42:39,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:42:39,584 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112243bdd4c1a8344ad7a9fe7f80ca24ead1_8422287be02eb151a34ae7fe752afee0 to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112243bdd4c1a8344ad7a9fe7f80ca24ead1_8422287be02eb151a34ae7fe752afee0 2024-11-22T03:42:39,585 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/.tmp/A/0720a04112f143309f37a2199a4a5f5a, store: [table=TestAcidGuarantees family=A region=8422287be02eb151a34ae7fe752afee0] 2024-11-22T03:42:39,585 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/.tmp/A/0720a04112f143309f37a2199a4a5f5a is 175, key is test_row_0/A:col10/1732246957167/Put/seqid=0 2024-11-22T03:42:39,588 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742356_1532 (size=31105) 2024-11-22T03:42:39,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-22T03:42:39,660 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:42:39,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37380 deadline: 1732247019658, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 2024-11-22T03:42:39,661 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:39,661 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:39,661 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:42:39,670 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:42:39,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37382 deadline: 1732247019669, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 2024-11-22T03:42:39,670 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:39,670 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:39,670 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:42:39,971 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:42:39,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37380 deadline: 1732247019969, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 2024-11-22T03:42:39,971 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:39,971 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:39,971 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:42:39,980 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:42:39,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37382 deadline: 1732247019978, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 2024-11-22T03:42:39,981 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:39,981 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:39,981 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:42:39,989 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=195, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/.tmp/A/0720a04112f143309f37a2199a4a5f5a 2024-11-22T03:42:39,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/.tmp/B/0448d51157614cff8abf978e06e9494f is 50, key is test_row_0/B:col10/1732246957167/Put/seqid=0 2024-11-22T03:42:39,998 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742357_1533 (size=12151) 2024-11-22T03:42:39,998 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=195 (bloomFilter=true), to=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/.tmp/B/0448d51157614cff8abf978e06e9494f 2024-11-22T03:42:40,003 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/.tmp/C/b73ed0bbb8174881a95580ceb54bf62b is 50, key is test_row_0/C:col10/1732246957167/Put/seqid=0 2024-11-22T03:42:40,013 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742358_1534 (size=12151) 2024-11-22T03:42:40,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-22T03:42:40,414 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=195 (bloomFilter=true), to=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/.tmp/C/b73ed0bbb8174881a95580ceb54bf62b 2024-11-22T03:42:40,418 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/.tmp/A/0720a04112f143309f37a2199a4a5f5a as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/A/0720a04112f143309f37a2199a4a5f5a 2024-11-22T03:42:40,421 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/A/0720a04112f143309f37a2199a4a5f5a, entries=150, sequenceid=195, filesize=30.4 K 2024-11-22T03:42:40,422 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/.tmp/B/0448d51157614cff8abf978e06e9494f as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/B/0448d51157614cff8abf978e06e9494f 2024-11-22T03:42:40,425 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/B/0448d51157614cff8abf978e06e9494f, entries=150, sequenceid=195, filesize=11.9 K 2024-11-22T03:42:40,426 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/.tmp/C/b73ed0bbb8174881a95580ceb54bf62b as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/C/b73ed0bbb8174881a95580ceb54bf62b 2024-11-22T03:42:40,429 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/C/b73ed0bbb8174881a95580ceb54bf62b, entries=150, sequenceid=195, filesize=11.9 K 2024-11-22T03:42:40,430 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for 8422287be02eb151a34ae7fe752afee0 in 1257ms, sequenceid=195, compaction requested=false 2024-11-22T03:42:40,430 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2603): Flush status journal for 8422287be02eb151a34ae7fe752afee0: 2024-11-22T03:42:40,430 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0. 2024-11-22T03:42:40,430 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=157 2024-11-22T03:42:40,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42627 {}] master.HMaster(4169): Remote procedure done, pid=157 2024-11-22T03:42:40,432 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=157, resume processing ppid=156 2024-11-22T03:42:40,432 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=157, ppid=156, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4100 sec 2024-11-22T03:42:40,434 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=156, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees in 1.4140 sec 2024-11-22T03:42:40,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(8855): Flush requested on 8422287be02eb151a34ae7fe752afee0 2024-11-22T03:42:40,481 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-22T03:42:40,481 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8422287be02eb151a34ae7fe752afee0 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-22T03:42:40,491 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411224b4be423161c4a0584d2b5cf54a51719_8422287be02eb151a34ae7fe752afee0 is 50, key is test_row_0/A:col10/1732246960480/Put/seqid=0 2024-11-22T03:42:40,498 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742359_1535 (size=14794) 2024-11-22T03:42:40,538 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:42:40,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37380 deadline: 1732247020534, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 2024-11-22T03:42:40,539 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:40,539 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:40,539 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:42:40,539 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:42:40,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37382 deadline: 1732247020535, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 2024-11-22T03:42:40,540 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:40,540 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:40,540 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:42:40,650 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:42:40,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37380 deadline: 1732247020648, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 2024-11-22T03:42:40,651 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:42:40,651 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:40,651 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:40,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37382 deadline: 1732247020648, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 2024-11-22T03:42:40,651 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:42:40,651 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:40,651 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:40,651 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:42:40,860 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:42:40,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37382 deadline: 1732247020858, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 2024-11-22T03:42:40,861 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:42:40,861 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:40,861 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:40,861 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:42:40,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37380 deadline: 1732247020858, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 2024-11-22T03:42:40,861 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:40,861 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:40,861 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:42:40,898 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:42:40,901 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411224b4be423161c4a0584d2b5cf54a51719_8422287be02eb151a34ae7fe752afee0 to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411224b4be423161c4a0584d2b5cf54a51719_8422287be02eb151a34ae7fe752afee0 2024-11-22T03:42:40,902 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/.tmp/A/013d0e162c6048a094d054d77649f0d6, store: [table=TestAcidGuarantees family=A region=8422287be02eb151a34ae7fe752afee0] 2024-11-22T03:42:40,902 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/.tmp/A/013d0e162c6048a094d054d77649f0d6 is 175, key is test_row_0/A:col10/1732246960480/Put/seqid=0 2024-11-22T03:42:40,905 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742360_1536 (size=39749) 2024-11-22T03:42:40,906 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=208, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/.tmp/A/013d0e162c6048a094d054d77649f0d6 2024-11-22T03:42:40,915 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/.tmp/B/6b3b676052d34707bccae42e76c62eb2 is 50, key is test_row_0/B:col10/1732246960480/Put/seqid=0 2024-11-22T03:42:40,922 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742361_1537 (size=12151) 2024-11-22T03:42:40,923 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=208 (bloomFilter=true), to=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/.tmp/B/6b3b676052d34707bccae42e76c62eb2 2024-11-22T03:42:40,928 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/.tmp/C/f2d89d2f42544991adb170025d27ad77 is 50, key is test_row_0/C:col10/1732246960480/Put/seqid=0 2024-11-22T03:42:40,931 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742362_1538 (size=12151) 2024-11-22T03:42:41,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-22T03:42:41,169 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-22T03:42:41,169 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42627 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-22T03:42:41,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42627 {}] procedure2.ProcedureExecutor(1139): Stored pid=158, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees 2024-11-22T03:42:41,171 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:42:41,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-22T03:42:41,171 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=158, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-22T03:42:41,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37382 deadline: 1732247021169, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 2024-11-22T03:42:41,171 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:41,171 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:42:41,171 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:41,171 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:42:41,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37380 deadline: 1732247021169, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 2024-11-22T03:42:41,171 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:41,171 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:41,171 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:42:41,172 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=158, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-22T03:42:41,172 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=159, ppid=158, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-22T03:42:41,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-22T03:42:41,323 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42545 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-22T03:42:41,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0. 2024-11-22T03:42:41,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0. as already flushing 2024-11-22T03:42:41,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0. 2024-11-22T03:42:41,324 ERROR [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => 8422287be02eb151a34ae7fe752afee0, NAME => 'TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:42:41,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => 8422287be02eb151a34ae7fe752afee0, NAME => 'TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:42:41,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42627 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8422287be02eb151a34ae7fe752afee0, NAME => 'TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8422287be02eb151a34ae7fe752afee0, NAME => 'TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:42:41,331 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=208 (bloomFilter=true), to=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/.tmp/C/f2d89d2f42544991adb170025d27ad77 2024-11-22T03:42:41,335 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/.tmp/A/013d0e162c6048a094d054d77649f0d6 as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/A/013d0e162c6048a094d054d77649f0d6 2024-11-22T03:42:41,337 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/A/013d0e162c6048a094d054d77649f0d6, entries=200, sequenceid=208, filesize=38.8 K 2024-11-22T03:42:41,337 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/.tmp/B/6b3b676052d34707bccae42e76c62eb2 as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/B/6b3b676052d34707bccae42e76c62eb2 2024-11-22T03:42:41,340 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/B/6b3b676052d34707bccae42e76c62eb2, entries=150, sequenceid=208, filesize=11.9 K 2024-11-22T03:42:41,341 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/.tmp/C/f2d89d2f42544991adb170025d27ad77 as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/C/f2d89d2f42544991adb170025d27ad77 2024-11-22T03:42:41,344 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/C/f2d89d2f42544991adb170025d27ad77, entries=150, sequenceid=208, filesize=11.9 K 2024-11-22T03:42:41,344 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=140.89 KB/144270 for 8422287be02eb151a34ae7fe752afee0 in 863ms, sequenceid=208, compaction requested=true 2024-11-22T03:42:41,345 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8422287be02eb151a34ae7fe752afee0: 2024-11-22T03:42:41,345 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8422287be02eb151a34ae7fe752afee0:A, priority=-2147483648, current under compaction store size is 1 2024-11-22T03:42:41,345 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T03:42:41,345 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T03:42:41,345 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8422287be02eb151a34ae7fe752afee0:B, priority=-2147483648, current under compaction store size is 2 2024-11-22T03:42:41,345 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T03:42:41,345 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T03:42:41,345 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8422287be02eb151a34ae7fe752afee0:C, priority=-2147483648, current under compaction store size is 3 2024-11-22T03:42:41,345 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T03:42:41,346 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102335 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T03:42:41,346 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.HStore(1541): 8422287be02eb151a34ae7fe752afee0/A is initiating minor compaction (all files) 2024-11-22T03:42:41,346 INFO [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8422287be02eb151a34ae7fe752afee0/A in TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0. 2024-11-22T03:42:41,346 INFO [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/A/d0625ea7b4304705aabef95e0c1ecfe5, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/A/0720a04112f143309f37a2199a4a5f5a, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/A/013d0e162c6048a094d054d77649f0d6] into tmpdir=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/.tmp, totalSize=99.9 K 2024-11-22T03:42:41,346 INFO [RS:0;c85114ed5096:42545-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0. 2024-11-22T03:42:41,346 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0. files: [hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/A/d0625ea7b4304705aabef95e0c1ecfe5, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/A/0720a04112f143309f37a2199a4a5f5a, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/A/013d0e162c6048a094d054d77649f0d6] 2024-11-22T03:42:41,346 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36829 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T03:42:41,346 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HStore(1541): 8422287be02eb151a34ae7fe752afee0/B is initiating minor compaction (all files) 2024-11-22T03:42:41,346 INFO [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8422287be02eb151a34ae7fe752afee0/B in TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0. 2024-11-22T03:42:41,346 INFO [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/B/05ad277a790e445eb8957f5e67a78ee7, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/B/0448d51157614cff8abf978e06e9494f, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/B/6b3b676052d34707bccae42e76c62eb2] into tmpdir=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/.tmp, totalSize=36.0 K 2024-11-22T03:42:41,347 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] compactions.Compactor(225): Compacting d0625ea7b4304705aabef95e0c1ecfe5, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=168, earliestPutTs=1732246956806 2024-11-22T03:42:41,347 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] compactions.Compactor(225): Compacting 05ad277a790e445eb8957f5e67a78ee7, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=168, earliestPutTs=1732246956806 2024-11-22T03:42:41,347 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] compactions.Compactor(225): Compacting 0448d51157614cff8abf978e06e9494f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1732246957144 2024-11-22T03:42:41,347 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0720a04112f143309f37a2199a4a5f5a, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1732246957144 2024-11-22T03:42:41,347 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] compactions.Compactor(225): Compacting 6b3b676052d34707bccae42e76c62eb2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1732246959335 2024-11-22T03:42:41,347 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] compactions.Compactor(225): Compacting 013d0e162c6048a094d054d77649f0d6, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1732246959335 2024-11-22T03:42:41,351 INFO [RS:0;c85114ed5096:42545-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=8422287be02eb151a34ae7fe752afee0] 2024-11-22T03:42:41,353 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411223b2c7109ea0846efbd9e9c21f45a6cbe_8422287be02eb151a34ae7fe752afee0 store=[table=TestAcidGuarantees family=A region=8422287be02eb151a34ae7fe752afee0] 2024-11-22T03:42:41,354 INFO [RS:0;c85114ed5096:42545-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8422287be02eb151a34ae7fe752afee0#B#compaction#457 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-22T03:42:41,354 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/.tmp/B/005ebc1f537144f38e392eb604d35bd0 is 50, key is test_row_0/B:col10/1732246960480/Put/seqid=0 2024-11-22T03:42:41,355 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411223b2c7109ea0846efbd9e9c21f45a6cbe_8422287be02eb151a34ae7fe752afee0, store=[table=TestAcidGuarantees family=A region=8422287be02eb151a34ae7fe752afee0] 2024-11-22T03:42:41,355 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411223b2c7109ea0846efbd9e9c21f45a6cbe_8422287be02eb151a34ae7fe752afee0 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=8422287be02eb151a34ae7fe752afee0] 2024-11-22T03:42:41,370 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742363_1539 (size=12629) 2024-11-22T03:42:41,378 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742364_1540 (size=4469) 2024-11-22T03:42:41,380 INFO [RS:0;c85114ed5096:42545-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8422287be02eb151a34ae7fe752afee0#A#compaction#456 average throughput is 0.84 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T03:42:41,381 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/.tmp/A/90b9a6a483f24988990f2a8c6116c3b0 is 175, key is test_row_0/A:col10/1732246960480/Put/seqid=0 2024-11-22T03:42:41,386 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742365_1541 (size=31583) 2024-11-22T03:42:41,476 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42545 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-22T03:42:41,476 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0. 2024-11-22T03:42:41,476 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2902): Flushing 8422287be02eb151a34ae7fe752afee0 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-22T03:42:41,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241122e6a3098f7b7947c4b0ec58ad09fef8fe_8422287be02eb151a34ae7fe752afee0 is 50, key is test_row_0/A:col10/1732246960522/Put/seqid=0 2024-11-22T03:42:41,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-22T03:42:41,492 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742366_1542 (size=12304) 2024-11-22T03:42:41,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:42:41,495 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241122e6a3098f7b7947c4b0ec58ad09fef8fe_8422287be02eb151a34ae7fe752afee0 to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241122e6a3098f7b7947c4b0ec58ad09fef8fe_8422287be02eb151a34ae7fe752afee0 2024-11-22T03:42:41,496 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/.tmp/A/ed064f0700954704a3076922b772cdc9, store: [table=TestAcidGuarantees family=A region=8422287be02eb151a34ae7fe752afee0] 2024-11-22T03:42:41,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/.tmp/A/ed064f0700954704a3076922b772cdc9 is 175, key is test_row_0/A:col10/1732246960522/Put/seqid=0 2024-11-22T03:42:41,502 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742367_1543 (size=31105) 2024-11-22T03:42:41,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(8855): Flush requested on 8422287be02eb151a34ae7fe752afee0 2024-11-22T03:42:41,682 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0. as already flushing 2024-11-22T03:42:41,701 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:42:41,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37380 deadline: 1732247021698, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 2024-11-22T03:42:41,701 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:41,701 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:41,701 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:42:41,705 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:42:41,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37382 deadline: 1732247021701, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 2024-11-22T03:42:41,705 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:41,705 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:41,705 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:42:41,773 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/.tmp/B/005ebc1f537144f38e392eb604d35bd0 as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/B/005ebc1f537144f38e392eb604d35bd0 2024-11-22T03:42:41,776 INFO [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8422287be02eb151a34ae7fe752afee0/B of 8422287be02eb151a34ae7fe752afee0 into 005ebc1f537144f38e392eb604d35bd0(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T03:42:41,777 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8422287be02eb151a34ae7fe752afee0: 2024-11-22T03:42:41,777 INFO [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., storeName=8422287be02eb151a34ae7fe752afee0/B, priority=13, startTime=1732246961345; duration=0sec 2024-11-22T03:42:41,777 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T03:42:41,777 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8422287be02eb151a34ae7fe752afee0:B 2024-11-22T03:42:41,777 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T03:42:41,778 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36829 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T03:42:41,778 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HStore(1541): 8422287be02eb151a34ae7fe752afee0/C is initiating minor compaction (all files) 2024-11-22T03:42:41,778 INFO [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8422287be02eb151a34ae7fe752afee0/C in TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0. 2024-11-22T03:42:41,778 INFO [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/C/848f862c06e847e683a59ec7e0c11d81, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/C/b73ed0bbb8174881a95580ceb54bf62b, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/C/f2d89d2f42544991adb170025d27ad77] into tmpdir=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/.tmp, totalSize=36.0 K 2024-11-22T03:42:41,778 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] compactions.Compactor(225): Compacting 848f862c06e847e683a59ec7e0c11d81, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=168, earliestPutTs=1732246956806 2024-11-22T03:42:41,778 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] compactions.Compactor(225): Compacting b73ed0bbb8174881a95580ceb54bf62b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1732246957144 2024-11-22T03:42:41,778 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] compactions.Compactor(225): Compacting f2d89d2f42544991adb170025d27ad77, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1732246959335 2024-11-22T03:42:41,784 INFO [RS:0;c85114ed5096:42545-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8422287be02eb151a34ae7fe752afee0#C#compaction#459 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T03:42:41,784 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/.tmp/C/c3203268621e47afbd8fa349f53e846d is 50, key is test_row_0/C:col10/1732246960480/Put/seqid=0 2024-11-22T03:42:41,789 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/.tmp/A/90b9a6a483f24988990f2a8c6116c3b0 as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/A/90b9a6a483f24988990f2a8c6116c3b0 2024-11-22T03:42:41,790 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742368_1544 (size=12629) 2024-11-22T03:42:41,794 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/.tmp/C/c3203268621e47afbd8fa349f53e846d as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/C/c3203268621e47afbd8fa349f53e846d 2024-11-22T03:42:41,794 INFO [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8422287be02eb151a34ae7fe752afee0/A of 8422287be02eb151a34ae7fe752afee0 into 90b9a6a483f24988990f2a8c6116c3b0(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T03:42:41,794 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8422287be02eb151a34ae7fe752afee0: 2024-11-22T03:42:41,794 INFO [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., storeName=8422287be02eb151a34ae7fe752afee0/A, priority=13, startTime=1732246961345; duration=0sec 2024-11-22T03:42:41,794 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T03:42:41,794 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8422287be02eb151a34ae7fe752afee0:A 2024-11-22T03:42:41,798 INFO [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8422287be02eb151a34ae7fe752afee0/C of 8422287be02eb151a34ae7fe752afee0 into c3203268621e47afbd8fa349f53e846d(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T03:42:41,798 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8422287be02eb151a34ae7fe752afee0: 2024-11-22T03:42:41,798 INFO [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., storeName=8422287be02eb151a34ae7fe752afee0/C, priority=13, startTime=1732246961345; duration=0sec 2024-11-22T03:42:41,798 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T03:42:41,798 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8422287be02eb151a34ae7fe752afee0:C 2024-11-22T03:42:41,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-22T03:42:41,810 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:42:41,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37380 deadline: 1732247021808, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 2024-11-22T03:42:41,811 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:41,811 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:42:41,811 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:41,811 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:42:41,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37382 deadline: 1732247021808, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 2024-11-22T03:42:41,811 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:41,811 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:41,811 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:42:41,903 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=232, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/.tmp/A/ed064f0700954704a3076922b772cdc9 2024-11-22T03:42:41,910 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/.tmp/B/a172ebbd77234473bc738fce310a34cd is 50, key is test_row_0/B:col10/1732246960522/Put/seqid=0 2024-11-22T03:42:41,915 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742369_1545 (size=12151) 2024-11-22T03:42:42,020 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:42:42,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37380 deadline: 1732247022018, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 2024-11-22T03:42:42,020 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:42,020 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:42:42,020 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:42,020 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:42:42,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37382 deadline: 1732247022018, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 2024-11-22T03:42:42,021 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:42,021 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:42,021 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:42:42,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-22T03:42:42,316 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=232 (bloomFilter=true), to=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/.tmp/B/a172ebbd77234473bc738fce310a34cd 2024-11-22T03:42:42,321 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/.tmp/C/2d6531b5a63e444f83ceef4ec84ec6ad is 50, key is test_row_0/C:col10/1732246960522/Put/seqid=0 2024-11-22T03:42:42,324 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742370_1546 (size=12151) 2024-11-22T03:42:42,331 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:42:42,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37380 deadline: 1732247022329, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 2024-11-22T03:42:42,331 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:42,331 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:42:42,331 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:42,331 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:42:42,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37382 deadline: 1732247022329, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 2024-11-22T03:42:42,331 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:42,331 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:42,331 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:42:42,724 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=232 (bloomFilter=true), to=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/.tmp/C/2d6531b5a63e444f83ceef4ec84ec6ad 2024-11-22T03:42:42,728 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/.tmp/A/ed064f0700954704a3076922b772cdc9 as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/A/ed064f0700954704a3076922b772cdc9 2024-11-22T03:42:42,731 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/A/ed064f0700954704a3076922b772cdc9, entries=150, sequenceid=232, filesize=30.4 K 2024-11-22T03:42:42,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/.tmp/B/a172ebbd77234473bc738fce310a34cd as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/B/a172ebbd77234473bc738fce310a34cd 2024-11-22T03:42:42,735 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/B/a172ebbd77234473bc738fce310a34cd, entries=150, sequenceid=232, filesize=11.9 K 2024-11-22T03:42:42,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/.tmp/C/2d6531b5a63e444f83ceef4ec84ec6ad as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/C/2d6531b5a63e444f83ceef4ec84ec6ad 2024-11-22T03:42:42,738 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/C/2d6531b5a63e444f83ceef4ec84ec6ad, entries=150, sequenceid=232, filesize=11.9 K 2024-11-22T03:42:42,739 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for 8422287be02eb151a34ae7fe752afee0 in 1263ms, sequenceid=232, compaction requested=false 2024-11-22T03:42:42,740 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2603): Flush status journal for 8422287be02eb151a34ae7fe752afee0: 2024-11-22T03:42:42,740 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0. 2024-11-22T03:42:42,740 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=159 2024-11-22T03:42:42,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42627 {}] master.HMaster(4169): Remote procedure done, pid=159 2024-11-22T03:42:42,742 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=159, resume processing ppid=158 2024-11-22T03:42:42,742 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=159, ppid=158, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5690 sec 2024-11-22T03:42:42,744 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=158, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees in 1.5740 sec 2024-11-22T03:42:42,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(8855): Flush requested on 8422287be02eb151a34ae7fe752afee0 2024-11-22T03:42:42,840 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-22T03:42:42,840 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8422287be02eb151a34ae7fe752afee0 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-22T03:42:42,846 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241122dfcbfb5b53974871ac7418c046dab708_8422287be02eb151a34ae7fe752afee0 is 50, key is test_row_0/A:col10/1732246961696/Put/seqid=0 2024-11-22T03:42:42,852 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742371_1547 (size=14794) 2024-11-22T03:42:42,879 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:42:42,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37382 deadline: 1732247022876, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 2024-11-22T03:42:42,880 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:42,880 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:42,880 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:42:42,883 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:42:42,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37380 deadline: 1732247022880, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 2024-11-22T03:42:42,884 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:42,884 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:42,884 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:42:42,991 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:42:42,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37382 deadline: 1732247022989, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 2024-11-22T03:42:42,991 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:42:42,991 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:42,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37380 deadline: 1732247022989, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 2024-11-22T03:42:42,991 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:42,991 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:42:42,992 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:42,992 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:42,992 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:42:43,202 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:42:43,202 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:42:43,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37380 deadline: 1732247023199, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 2024-11-22T03:42:43,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37382 deadline: 1732247023199, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 2024-11-22T03:42:43,203 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:43,203 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:43,203 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:42:43,203 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:43,203 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:43,203 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:42:43,252 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:42:43,254 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241122dfcbfb5b53974871ac7418c046dab708_8422287be02eb151a34ae7fe752afee0 to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241122dfcbfb5b53974871ac7418c046dab708_8422287be02eb151a34ae7fe752afee0 2024-11-22T03:42:43,255 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/.tmp/A/0c138a3b9dd54e8d9caa9b80bc28ce10, store: [table=TestAcidGuarantees family=A region=8422287be02eb151a34ae7fe752afee0] 2024-11-22T03:42:43,255 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/.tmp/A/0c138a3b9dd54e8d9caa9b80bc28ce10 is 175, key is test_row_0/A:col10/1732246961696/Put/seqid=0 2024-11-22T03:42:43,265 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742372_1548 (size=39749) 2024-11-22T03:42:43,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-22T03:42:43,319 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-22T03:42:43,319 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42627 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-22T03:42:43,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42627 {}] procedure2.ProcedureExecutor(1139): Stored pid=160, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees 2024-11-22T03:42:43,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-22T03:42:43,321 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=160, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-22T03:42:43,321 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=160, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-22T03:42:43,322 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=161, ppid=160, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-22T03:42:43,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-22T03:42:43,473 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42545 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-22T03:42:43,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0. 2024-11-22T03:42:43,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0. as already flushing 2024-11-22T03:42:43,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0. 2024-11-22T03:42:43,474 ERROR [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 8422287be02eb151a34ae7fe752afee0, NAME => 'TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:42:43,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 8422287be02eb151a34ae7fe752afee0, NAME => 'TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:42:43,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42627 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8422287be02eb151a34ae7fe752afee0, NAME => 'TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8422287be02eb151a34ae7fe752afee0, NAME => 'TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:42:43,511 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:42:43,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37380 deadline: 1732247023508, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 2024-11-22T03:42:43,511 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:42:43,511 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:43,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37382 deadline: 1732247023508, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 2024-11-22T03:42:43,511 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:43,511 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:42:43,512 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:43,512 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:43,512 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:42:43,626 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42545 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-22T03:42:43,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0. 2024-11-22T03:42:43,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0. as already flushing 2024-11-22T03:42:43,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0. 2024-11-22T03:42:43,626 ERROR [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 8422287be02eb151a34ae7fe752afee0, NAME => 'TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:42:43,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 8422287be02eb151a34ae7fe752afee0, NAME => 'TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:42:43,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42627 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8422287be02eb151a34ae7fe752afee0, NAME => 'TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8422287be02eb151a34ae7fe752afee0, NAME => 'TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:42:43,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-22T03:42:43,666 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=249, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/.tmp/A/0c138a3b9dd54e8d9caa9b80bc28ce10 2024-11-22T03:42:43,671 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/.tmp/B/6c491051f018435fa87469695882acca is 50, key is test_row_0/B:col10/1732246961696/Put/seqid=0 2024-11-22T03:42:43,673 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742373_1549 (size=12151) 2024-11-22T03:42:43,778 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42545 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-22T03:42:43,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0. 2024-11-22T03:42:43,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0. as already flushing 2024-11-22T03:42:43,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0. 2024-11-22T03:42:43,779 ERROR [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 8422287be02eb151a34ae7fe752afee0, NAME => 'TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:42:43,779 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 8422287be02eb151a34ae7fe752afee0, NAME => 'TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:42:43,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42627 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8422287be02eb151a34ae7fe752afee0, NAME => 'TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8422287be02eb151a34ae7fe752afee0, NAME => 'TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:42:43,931 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42545 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-22T03:42:43,931 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0. 2024-11-22T03:42:43,931 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0. as already flushing 2024-11-22T03:42:43,931 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0. 2024-11-22T03:42:43,931 ERROR [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 8422287be02eb151a34ae7fe752afee0, NAME => 'TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:42:43,931 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 8422287be02eb151a34ae7fe752afee0, NAME => 'TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:42:43,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42627 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8422287be02eb151a34ae7fe752afee0, NAME => 'TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8422287be02eb151a34ae7fe752afee0, NAME => 'TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:42:43,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-22T03:42:44,019 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:42:44,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37380 deadline: 1732247024018, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 2024-11-22T03:42:44,019 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:42:44,020 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:44,020 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:44,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37382 deadline: 1732247024018, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 2024-11-22T03:42:44,020 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:42:44,020 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:44,020 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:44,020 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:42:44,074 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=249 (bloomFilter=true), to=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/.tmp/B/6c491051f018435fa87469695882acca 2024-11-22T03:42:44,082 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/.tmp/C/cda48792dad4403c97cf102f1aaee83f is 50, key is test_row_0/C:col10/1732246961696/Put/seqid=0 2024-11-22T03:42:44,083 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42545 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-22T03:42:44,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0. 2024-11-22T03:42:44,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0. as already flushing 2024-11-22T03:42:44,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0. 2024-11-22T03:42:44,083 ERROR [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 8422287be02eb151a34ae7fe752afee0, NAME => 'TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:42:44,084 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 8422287be02eb151a34ae7fe752afee0, NAME => 'TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:42:44,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42627 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8422287be02eb151a34ae7fe752afee0, NAME => 'TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8422287be02eb151a34ae7fe752afee0, NAME => 'TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T03:42:44,086 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742374_1550 (size=12151) 2024-11-22T03:42:44,087 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=249 (bloomFilter=true), to=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/.tmp/C/cda48792dad4403c97cf102f1aaee83f 2024-11-22T03:42:44,090 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/.tmp/A/0c138a3b9dd54e8d9caa9b80bc28ce10 as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/A/0c138a3b9dd54e8d9caa9b80bc28ce10 2024-11-22T03:42:44,092 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/A/0c138a3b9dd54e8d9caa9b80bc28ce10, entries=200, sequenceid=249, filesize=38.8 K 2024-11-22T03:42:44,093 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/.tmp/B/6c491051f018435fa87469695882acca as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/B/6c491051f018435fa87469695882acca 2024-11-22T03:42:44,095 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/B/6c491051f018435fa87469695882acca, entries=150, sequenceid=249, filesize=11.9 K 2024-11-22T03:42:44,096 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/.tmp/C/cda48792dad4403c97cf102f1aaee83f as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/C/cda48792dad4403c97cf102f1aaee83f 2024-11-22T03:42:44,099 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/C/cda48792dad4403c97cf102f1aaee83f, entries=150, sequenceid=249, filesize=11.9 K 2024-11-22T03:42:44,100 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for 8422287be02eb151a34ae7fe752afee0 in 1260ms, sequenceid=249, compaction requested=true 2024-11-22T03:42:44,100 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8422287be02eb151a34ae7fe752afee0: 2024-11-22T03:42:44,100 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8422287be02eb151a34ae7fe752afee0:A, priority=-2147483648, current under compaction store size is 1 2024-11-22T03:42:44,100 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T03:42:44,100 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T03:42:44,100 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8422287be02eb151a34ae7fe752afee0:B, priority=-2147483648, current under compaction store size is 2 2024-11-22T03:42:44,100 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T03:42:44,100 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8422287be02eb151a34ae7fe752afee0:C, priority=-2147483648, current under compaction store size is 3 2024-11-22T03:42:44,100 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-22T03:42:44,100 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T03:42:44,103 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102437 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T03:42:44,103 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36931 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T03:42:44,103 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HStore(1541): 8422287be02eb151a34ae7fe752afee0/A is initiating minor compaction (all files) 2024-11-22T03:42:44,103 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.HStore(1541): 8422287be02eb151a34ae7fe752afee0/B is initiating minor compaction (all files) 2024-11-22T03:42:44,103 INFO [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8422287be02eb151a34ae7fe752afee0/A in TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0. 2024-11-22T03:42:44,103 INFO [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8422287be02eb151a34ae7fe752afee0/B in TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0. 2024-11-22T03:42:44,103 INFO [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/A/90b9a6a483f24988990f2a8c6116c3b0, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/A/ed064f0700954704a3076922b772cdc9, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/A/0c138a3b9dd54e8d9caa9b80bc28ce10] into tmpdir=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/.tmp, totalSize=100.0 K 2024-11-22T03:42:44,103 INFO [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/B/005ebc1f537144f38e392eb604d35bd0, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/B/a172ebbd77234473bc738fce310a34cd, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/B/6c491051f018435fa87469695882acca] into tmpdir=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/.tmp, totalSize=36.1 K 2024-11-22T03:42:44,103 INFO [RS:0;c85114ed5096:42545-longCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0. 2024-11-22T03:42:44,103 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0. files: [hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/A/90b9a6a483f24988990f2a8c6116c3b0, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/A/ed064f0700954704a3076922b772cdc9, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/A/0c138a3b9dd54e8d9caa9b80bc28ce10] 2024-11-22T03:42:44,103 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] compactions.Compactor(225): Compacting 005ebc1f537144f38e392eb604d35bd0, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1732246959335 2024-11-22T03:42:44,103 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] compactions.Compactor(225): Compacting 90b9a6a483f24988990f2a8c6116c3b0, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1732246959335 2024-11-22T03:42:44,104 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] compactions.Compactor(225): Compacting ed064f0700954704a3076922b772cdc9, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=232, earliestPutTs=1732246960521 2024-11-22T03:42:44,104 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] compactions.Compactor(225): Compacting a172ebbd77234473bc738fce310a34cd, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=232, earliestPutTs=1732246960521 2024-11-22T03:42:44,104 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] compactions.Compactor(225): Compacting 0c138a3b9dd54e8d9caa9b80bc28ce10, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=249, earliestPutTs=1732246961696 2024-11-22T03:42:44,104 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6c491051f018435fa87469695882acca, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=249, earliestPutTs=1732246961696 2024-11-22T03:42:44,114 INFO [RS:0;c85114ed5096:42545-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8422287be02eb151a34ae7fe752afee0#B#compaction#465 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T03:42:44,114 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/.tmp/B/abe6c4acb5324a9dbb6c8c4faada4a10 is 50, key is test_row_0/B:col10/1732246961696/Put/seqid=0 2024-11-22T03:42:44,118 INFO [RS:0;c85114ed5096:42545-longCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=8422287be02eb151a34ae7fe752afee0] 2024-11-22T03:42:44,120 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742375_1551 (size=12731) 2024-11-22T03:42:44,120 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241122356cc98bbb674ee5a1d45c6ff5dd40b5_8422287be02eb151a34ae7fe752afee0 store=[table=TestAcidGuarantees family=A region=8422287be02eb151a34ae7fe752afee0] 2024-11-22T03:42:44,122 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241122356cc98bbb674ee5a1d45c6ff5dd40b5_8422287be02eb151a34ae7fe752afee0, store=[table=TestAcidGuarantees family=A region=8422287be02eb151a34ae7fe752afee0] 2024-11-22T03:42:44,123 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241122356cc98bbb674ee5a1d45c6ff5dd40b5_8422287be02eb151a34ae7fe752afee0 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=8422287be02eb151a34ae7fe752afee0] 2024-11-22T03:42:44,126 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742376_1552 (size=4469) 2024-11-22T03:42:44,236 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42545 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-22T03:42:44,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0. 2024-11-22T03:42:44,236 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2902): Flushing 8422287be02eb151a34ae7fe752afee0 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-22T03:42:44,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241122ed86cd200aeb4043b69dd85f13411b12_8422287be02eb151a34ae7fe752afee0 is 50, key is test_row_0/A:col10/1732246962875/Put/seqid=0 2024-11-22T03:42:44,245 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742377_1553 (size=12454) 2024-11-22T03:42:44,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:42:44,248 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241122ed86cd200aeb4043b69dd85f13411b12_8422287be02eb151a34ae7fe752afee0 to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241122ed86cd200aeb4043b69dd85f13411b12_8422287be02eb151a34ae7fe752afee0 2024-11-22T03:42:44,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/.tmp/A/bce03a2cd14944ac891aa8baac4adbd5, store: [table=TestAcidGuarantees family=A region=8422287be02eb151a34ae7fe752afee0] 2024-11-22T03:42:44,249 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/.tmp/A/bce03a2cd14944ac891aa8baac4adbd5 is 175, key is test_row_0/A:col10/1732246962875/Put/seqid=0 2024-11-22T03:42:44,252 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742378_1554 (size=31255) 2024-11-22T03:42:44,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-22T03:42:44,524 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/.tmp/B/abe6c4acb5324a9dbb6c8c4faada4a10 as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/B/abe6c4acb5324a9dbb6c8c4faada4a10 2024-11-22T03:42:44,527 INFO [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8422287be02eb151a34ae7fe752afee0/B of 8422287be02eb151a34ae7fe752afee0 into abe6c4acb5324a9dbb6c8c4faada4a10(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T03:42:44,527 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8422287be02eb151a34ae7fe752afee0: 2024-11-22T03:42:44,527 INFO [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., storeName=8422287be02eb151a34ae7fe752afee0/B, priority=13, startTime=1732246964100; duration=0sec 2024-11-22T03:42:44,527 INFO [RS:0;c85114ed5096:42545-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8422287be02eb151a34ae7fe752afee0#A#compaction#466 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T03:42:44,527 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T03:42:44,527 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8422287be02eb151a34ae7fe752afee0:B 2024-11-22T03:42:44,527 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T03:42:44,528 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/.tmp/A/02e7cbda247a49cc87aa17302630af4f is 175, key is test_row_0/A:col10/1732246961696/Put/seqid=0 2024-11-22T03:42:44,528 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36931 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T03:42:44,528 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.HStore(1541): 8422287be02eb151a34ae7fe752afee0/C is initiating minor compaction (all files) 2024-11-22T03:42:44,528 INFO [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8422287be02eb151a34ae7fe752afee0/C in TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0. 2024-11-22T03:42:44,529 INFO [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/C/c3203268621e47afbd8fa349f53e846d, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/C/2d6531b5a63e444f83ceef4ec84ec6ad, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/C/cda48792dad4403c97cf102f1aaee83f] into tmpdir=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/.tmp, totalSize=36.1 K 2024-11-22T03:42:44,529 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] compactions.Compactor(225): Compacting c3203268621e47afbd8fa349f53e846d, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1732246959335 2024-11-22T03:42:44,530 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2d6531b5a63e444f83ceef4ec84ec6ad, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=232, earliestPutTs=1732246960521 2024-11-22T03:42:44,530 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] compactions.Compactor(225): Compacting cda48792dad4403c97cf102f1aaee83f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=249, earliestPutTs=1732246961696 2024-11-22T03:42:44,531 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742379_1555 (size=31685) 2024-11-22T03:42:44,534 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/.tmp/A/02e7cbda247a49cc87aa17302630af4f as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/A/02e7cbda247a49cc87aa17302630af4f 2024-11-22T03:42:44,536 INFO [RS:0;c85114ed5096:42545-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8422287be02eb151a34ae7fe752afee0#C#compaction#468 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T03:42:44,536 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/.tmp/C/71419a78f75042e6893695fa66f0247a is 50, key is test_row_0/C:col10/1732246961696/Put/seqid=0 2024-11-22T03:42:44,538 INFO [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8422287be02eb151a34ae7fe752afee0/A of 8422287be02eb151a34ae7fe752afee0 into 02e7cbda247a49cc87aa17302630af4f(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T03:42:44,538 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8422287be02eb151a34ae7fe752afee0: 2024-11-22T03:42:44,538 INFO [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., storeName=8422287be02eb151a34ae7fe752afee0/A, priority=13, startTime=1732246964100; duration=0sec 2024-11-22T03:42:44,538 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T03:42:44,538 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8422287be02eb151a34ae7fe752afee0:A 2024-11-22T03:42:44,539 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742380_1556 (size=12731) 2024-11-22T03:42:44,652 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=271, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/.tmp/A/bce03a2cd14944ac891aa8baac4adbd5 2024-11-22T03:42:44,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/.tmp/B/245cfa1382974849bdd2e12f1e7d3a48 is 50, key is test_row_0/B:col10/1732246962875/Put/seqid=0 2024-11-22T03:42:44,660 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742381_1557 (size=12301) 2024-11-22T03:42:44,823 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-22T03:42:44,943 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/.tmp/C/71419a78f75042e6893695fa66f0247a as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/C/71419a78f75042e6893695fa66f0247a 2024-11-22T03:42:44,946 INFO [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8422287be02eb151a34ae7fe752afee0/C of 8422287be02eb151a34ae7fe752afee0 into 71419a78f75042e6893695fa66f0247a(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T03:42:44,946 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8422287be02eb151a34ae7fe752afee0: 2024-11-22T03:42:44,946 INFO [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., storeName=8422287be02eb151a34ae7fe752afee0/C, priority=13, startTime=1732246964100; duration=0sec 2024-11-22T03:42:44,946 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T03:42:44,946 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8422287be02eb151a34ae7fe752afee0:C 2024-11-22T03:42:45,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(8855): Flush requested on 8422287be02eb151a34ae7fe752afee0 2024-11-22T03:42:45,030 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0. as already flushing 2024-11-22T03:42:45,047 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:42:45,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37382 deadline: 1732247025044, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 2024-11-22T03:42:45,047 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:45,047 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:45,047 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:42:45,047 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:42:45,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 169 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37380 deadline: 1732247025045, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 2024-11-22T03:42:45,048 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:45,048 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:45,048 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:42:45,061 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=271 (bloomFilter=true), to=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/.tmp/B/245cfa1382974849bdd2e12f1e7d3a48 2024-11-22T03:42:45,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/.tmp/C/22511ae460cf4409ad7bfc2f0def223c is 50, key is test_row_0/C:col10/1732246962875/Put/seqid=0 2024-11-22T03:42:45,069 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742382_1558 (size=12301) 2024-11-22T03:42:45,159 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:42:45,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37382 deadline: 1732247025158, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 2024-11-22T03:42:45,160 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:42:45,160 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:45,160 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:45,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 170 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37380 deadline: 1732247025158, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 2024-11-22T03:42:45,160 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:42:45,160 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:45,160 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:45,160 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:42:45,369 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:42:45,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37382 deadline: 1732247025369, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 2024-11-22T03:42:45,370 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:45,370 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:45,370 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:42:45,371 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:42:45,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 171 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37380 deadline: 1732247025369, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 2024-11-22T03:42:45,371 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:45,371 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:45,371 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:42:45,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-22T03:42:45,469 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=271 (bloomFilter=true), to=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/.tmp/C/22511ae460cf4409ad7bfc2f0def223c 2024-11-22T03:42:45,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/.tmp/A/bce03a2cd14944ac891aa8baac4adbd5 as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/A/bce03a2cd14944ac891aa8baac4adbd5 2024-11-22T03:42:45,475 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/A/bce03a2cd14944ac891aa8baac4adbd5, entries=150, sequenceid=271, filesize=30.5 K 2024-11-22T03:42:45,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/.tmp/B/245cfa1382974849bdd2e12f1e7d3a48 as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/B/245cfa1382974849bdd2e12f1e7d3a48 2024-11-22T03:42:45,478 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/B/245cfa1382974849bdd2e12f1e7d3a48, entries=150, sequenceid=271, filesize=12.0 K 2024-11-22T03:42:45,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/.tmp/C/22511ae460cf4409ad7bfc2f0def223c as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/C/22511ae460cf4409ad7bfc2f0def223c 2024-11-22T03:42:45,481 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/C/22511ae460cf4409ad7bfc2f0def223c, entries=150, sequenceid=271, filesize=12.0 K 2024-11-22T03:42:45,481 INFO [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=73.80 KB/75570 for 8422287be02eb151a34ae7fe752afee0 in 1245ms, sequenceid=271, compaction requested=false 2024-11-22T03:42:45,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2603): Flush status journal for 8422287be02eb151a34ae7fe752afee0: 2024-11-22T03:42:45,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0. 2024-11-22T03:42:45,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c85114ed5096:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=161 2024-11-22T03:42:45,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42627 {}] master.HMaster(4169): Remote procedure done, pid=161 2024-11-22T03:42:45,484 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=161, resume processing ppid=160 2024-11-22T03:42:45,484 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=161, ppid=160, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1610 sec 2024-11-22T03:42:45,485 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=160, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees in 2.1650 sec 2024-11-22T03:42:45,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(8855): Flush requested on 8422287be02eb151a34ae7fe752afee0 2024-11-22T03:42:45,680 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-22T03:42:45,680 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8422287be02eb151a34ae7fe752afee0 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-22T03:42:45,686 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241122c44edc5f1abe4b6ea15b45c26929e141_8422287be02eb151a34ae7fe752afee0 is 50, key is test_row_0/A:col10/1732246965035/Put/seqid=0 2024-11-22T03:42:45,689 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742383_1559 (size=14994) 2024-11-22T03:42:45,689 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:42:45,693 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241122c44edc5f1abe4b6ea15b45c26929e141_8422287be02eb151a34ae7fe752afee0 to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241122c44edc5f1abe4b6ea15b45c26929e141_8422287be02eb151a34ae7fe752afee0 2024-11-22T03:42:45,694 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/.tmp/A/aaa6f22fd13349bdae1afebf8ac400d5, store: [table=TestAcidGuarantees family=A region=8422287be02eb151a34ae7fe752afee0] 2024-11-22T03:42:45,694 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/.tmp/A/aaa6f22fd13349bdae1afebf8ac400d5 is 175, key is test_row_0/A:col10/1732246965035/Put/seqid=0 2024-11-22T03:42:45,697 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742384_1560 (size=39949) 2024-11-22T03:42:45,715 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:42:45,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37382 deadline: 1732247025712, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 2024-11-22T03:42:45,716 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:45,716 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:45,716 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:42:45,718 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:42:45,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 182 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37380 deadline: 1732247025715, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 2024-11-22T03:42:45,718 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:45,718 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:45,718 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:42:45,822 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:42:45,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37382 deadline: 1732247025818, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 2024-11-22T03:42:45,822 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:45,822 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:45,822 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:42:45,831 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:42:45,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 183 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37380 deadline: 1732247025829, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 2024-11-22T03:42:45,831 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:45,831 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:45,831 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:42:46,030 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:42:46,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37382 deadline: 1732247026028, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 2024-11-22T03:42:46,031 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:46,031 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:46,031 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:42:46,039 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:42:46,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 184 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37380 deadline: 1732247026038, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 2024-11-22T03:42:46,040 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:46,040 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:46,040 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:42:46,098 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=290, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/.tmp/A/aaa6f22fd13349bdae1afebf8ac400d5 2024-11-22T03:42:46,102 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/.tmp/B/7e89c9e9c59142d8a597564f6b374f21 is 50, key is test_row_0/B:col10/1732246965035/Put/seqid=0 2024-11-22T03:42:46,105 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742385_1561 (size=12301) 2024-11-22T03:42:46,340 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:42:46,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37382 deadline: 1732247026339, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 2024-11-22T03:42:46,341 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:46,341 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:46,341 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:42:46,351 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:42:46,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 185 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37380 deadline: 1732247026349, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 2024-11-22T03:42:46,351 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:46,351 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:46,351 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:42:46,506 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=290 (bloomFilter=true), to=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/.tmp/B/7e89c9e9c59142d8a597564f6b374f21 2024-11-22T03:42:46,511 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/.tmp/C/a22a4f9d75cf44aca92f651d71df8fbd is 50, key is test_row_0/C:col10/1732246965035/Put/seqid=0 2024-11-22T03:42:46,515 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742386_1562 (size=12301) 2024-11-22T03:42:46,529 INFO [Thread-2195 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2195. 2024-11-22T03:42:46,529 INFO [Thread-2193 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2193. 2024-11-22T03:42:46,529 DEBUG [Thread-2193 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-22T03:42:46,529 DEBUG [Thread-2195 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-22T03:42:46,529 DEBUG [Thread-2193 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T03:42:46,529 DEBUG [Thread-2195 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T03:42:46,529 DEBUG [Thread-2195 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T03:42:46,529 DEBUG [Thread-2193 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T03:42:46,529 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-22T03:42:46,530 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-22T03:42:46,530 INFO [Thread-2191 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2191. 2024-11-22T03:42:46,530 DEBUG [Thread-2191 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-22T03:42:46,530 DEBUG [Thread-2191 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T03:42:46,530 DEBUG [Thread-2191 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T03:42:46,530 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-22T03:42:46,532 INFO [Thread-2199 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2199. 2024-11-22T03:42:46,532 DEBUG [Thread-2199 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-22T03:42:46,532 DEBUG [Thread-2199 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T03:42:46,532 DEBUG [Thread-2199 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T03:42:46,532 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-22T03:42:46,533 INFO [Thread-2197 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2197. 2024-11-22T03:42:46,533 DEBUG [Thread-2197 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-22T03:42:46,533 DEBUG [Thread-2197 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T03:42:46,533 DEBUG [Thread-2197 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T03:42:46,533 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-22T03:42:46,850 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:42:46,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37382 deadline: 1732247026849, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 2024-11-22T03:42:46,851 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:46,851 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:46,851 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:42:46,859 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T03:42:46,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] ipc.CallRunner(138): callId: 186 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37380 deadline: 1732247026859, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 2024-11-22T03:42:46,860 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 , the old value is region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:46,861 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T03:42:46,861 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., hostname=c85114ed5096,42545,1732246786482, seqNum=5 because the exception is null or not the one we care about 2024-11-22T03:42:46,916 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=290 (bloomFilter=true), to=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/.tmp/C/a22a4f9d75cf44aca92f651d71df8fbd 2024-11-22T03:42:46,924 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/.tmp/A/aaa6f22fd13349bdae1afebf8ac400d5 as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/A/aaa6f22fd13349bdae1afebf8ac400d5 2024-11-22T03:42:46,929 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/A/aaa6f22fd13349bdae1afebf8ac400d5, entries=200, sequenceid=290, filesize=39.0 K 2024-11-22T03:42:46,930 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/.tmp/B/7e89c9e9c59142d8a597564f6b374f21 as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/B/7e89c9e9c59142d8a597564f6b374f21 2024-11-22T03:42:46,934 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/B/7e89c9e9c59142d8a597564f6b374f21, entries=150, sequenceid=290, filesize=12.0 K 2024-11-22T03:42:46,934 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/.tmp/C/a22a4f9d75cf44aca92f651d71df8fbd as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/C/a22a4f9d75cf44aca92f651d71df8fbd 2024-11-22T03:42:46,937 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/C/a22a4f9d75cf44aca92f651d71df8fbd, entries=150, sequenceid=290, filesize=12.0 K 2024-11-22T03:42:46,938 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=114.05 KB/116790 for 8422287be02eb151a34ae7fe752afee0 in 1258ms, sequenceid=290, compaction requested=true 2024-11-22T03:42:46,938 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8422287be02eb151a34ae7fe752afee0: 2024-11-22T03:42:46,938 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8422287be02eb151a34ae7fe752afee0:A, priority=-2147483648, current under compaction store size is 1 2024-11-22T03:42:46,938 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T03:42:46,938 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8422287be02eb151a34ae7fe752afee0:B, priority=-2147483648, current under compaction store size is 2 2024-11-22T03:42:46,938 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T03:42:46,938 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T03:42:46,938 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8422287be02eb151a34ae7fe752afee0:C, priority=-2147483648, current under compaction store size is 3 2024-11-22T03:42:46,938 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T03:42:46,938 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T03:42:46,939 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102889 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T03:42:46,939 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37333 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T03:42:46,939 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HStore(1541): 8422287be02eb151a34ae7fe752afee0/B is initiating minor compaction (all files) 2024-11-22T03:42:46,939 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.HStore(1541): 8422287be02eb151a34ae7fe752afee0/A is initiating minor compaction (all files) 2024-11-22T03:42:46,939 INFO [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8422287be02eb151a34ae7fe752afee0/A in TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0. 2024-11-22T03:42:46,939 INFO [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8422287be02eb151a34ae7fe752afee0/B in TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0. 2024-11-22T03:42:46,939 INFO [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/B/abe6c4acb5324a9dbb6c8c4faada4a10, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/B/245cfa1382974849bdd2e12f1e7d3a48, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/B/7e89c9e9c59142d8a597564f6b374f21] into tmpdir=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/.tmp, totalSize=36.5 K 2024-11-22T03:42:46,939 INFO [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/A/02e7cbda247a49cc87aa17302630af4f, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/A/bce03a2cd14944ac891aa8baac4adbd5, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/A/aaa6f22fd13349bdae1afebf8ac400d5] into tmpdir=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/.tmp, totalSize=100.5 K 2024-11-22T03:42:46,939 INFO [RS:0;c85114ed5096:42545-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0. 2024-11-22T03:42:46,939 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0. files: [hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/A/02e7cbda247a49cc87aa17302630af4f, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/A/bce03a2cd14944ac891aa8baac4adbd5, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/A/aaa6f22fd13349bdae1afebf8ac400d5] 2024-11-22T03:42:46,939 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] compactions.Compactor(225): Compacting abe6c4acb5324a9dbb6c8c4faada4a10, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=249, earliestPutTs=1732246961696 2024-11-22T03:42:46,939 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] compactions.Compactor(225): Compacting 02e7cbda247a49cc87aa17302630af4f, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=249, earliestPutTs=1732246961696 2024-11-22T03:42:46,939 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] compactions.Compactor(225): Compacting 245cfa1382974849bdd2e12f1e7d3a48, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=271, earliestPutTs=1732246962861 2024-11-22T03:42:46,939 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] compactions.Compactor(225): Compacting bce03a2cd14944ac891aa8baac4adbd5, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=271, earliestPutTs=1732246962861 2024-11-22T03:42:46,940 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] compactions.Compactor(225): Compacting 7e89c9e9c59142d8a597564f6b374f21, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=290, earliestPutTs=1732246965035 2024-11-22T03:42:46,940 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] compactions.Compactor(225): Compacting aaa6f22fd13349bdae1afebf8ac400d5, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=290, earliestPutTs=1732246965035 2024-11-22T03:42:46,946 INFO [RS:0;c85114ed5096:42545-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8422287be02eb151a34ae7fe752afee0#B#compaction#474 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T03:42:46,947 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/.tmp/B/48989c538a2f433ba12f955260d8a700 is 50, key is test_row_0/B:col10/1732246965035/Put/seqid=0 2024-11-22T03:42:46,948 INFO [RS:0;c85114ed5096:42545-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=8422287be02eb151a34ae7fe752afee0] 2024-11-22T03:42:46,949 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411224fc6b2f3740a4d178929a40014645de2_8422287be02eb151a34ae7fe752afee0 store=[table=TestAcidGuarantees family=A region=8422287be02eb151a34ae7fe752afee0] 2024-11-22T03:42:46,950 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742387_1563 (size=12983) 2024-11-22T03:42:46,959 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411224fc6b2f3740a4d178929a40014645de2_8422287be02eb151a34ae7fe752afee0, store=[table=TestAcidGuarantees family=A region=8422287be02eb151a34ae7fe752afee0] 2024-11-22T03:42:46,960 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411224fc6b2f3740a4d178929a40014645de2_8422287be02eb151a34ae7fe752afee0 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=8422287be02eb151a34ae7fe752afee0] 2024-11-22T03:42:46,963 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742388_1564 (size=4469) 2024-11-22T03:42:47,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42545 {}] regionserver.HRegion(8855): Flush requested on 8422287be02eb151a34ae7fe752afee0 2024-11-22T03:42:47,100 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-22T03:42:47,101 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8422287be02eb151a34ae7fe752afee0 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2024-11-22T03:42:47,101 INFO [Thread-2184 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2184. 2024-11-22T03:42:47,101 DEBUG [Thread-2184 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-22T03:42:47,101 DEBUG [Thread-2184 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T03:42:47,101 DEBUG [Thread-2184 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T03:42:47,101 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-22T03:42:47,107 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241122d33603a4211b4032a54b37a17f0b7246_8422287be02eb151a34ae7fe752afee0 is 50, key is test_row_0/A:col10/1732246965714/Put/seqid=0 2024-11-22T03:42:47,111 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742389_1565 (size=12454) 2024-11-22T03:42:47,140 INFO [Thread-2186 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2186. 2024-11-22T03:42:47,140 DEBUG [Thread-2186 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-22T03:42:47,140 DEBUG [Thread-2186 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T03:42:47,140 DEBUG [Thread-2186 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T03:42:47,140 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-22T03:42:47,210 INFO [Thread-2180 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2180. 2024-11-22T03:42:47,210 DEBUG [Thread-2180 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-22T03:42:47,210 DEBUG [Thread-2180 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T03:42:47,211 DEBUG [Thread-2180 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T03:42:47,211 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-22T03:42:47,356 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/.tmp/B/48989c538a2f433ba12f955260d8a700 as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/B/48989c538a2f433ba12f955260d8a700 2024-11-22T03:42:47,361 INFO [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8422287be02eb151a34ae7fe752afee0/B of 8422287be02eb151a34ae7fe752afee0 into 48989c538a2f433ba12f955260d8a700(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T03:42:47,361 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8422287be02eb151a34ae7fe752afee0: 2024-11-22T03:42:47,361 INFO [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., storeName=8422287be02eb151a34ae7fe752afee0/B, priority=13, startTime=1732246966938; duration=0sec 2024-11-22T03:42:47,361 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T03:42:47,361 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8422287be02eb151a34ae7fe752afee0:B 2024-11-22T03:42:47,361 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T03:42:47,362 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37333 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T03:42:47,362 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HStore(1541): 8422287be02eb151a34ae7fe752afee0/C is initiating minor compaction (all files) 2024-11-22T03:42:47,363 INFO [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8422287be02eb151a34ae7fe752afee0/C in TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0. 2024-11-22T03:42:47,363 INFO [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/C/71419a78f75042e6893695fa66f0247a, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/C/22511ae460cf4409ad7bfc2f0def223c, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/C/a22a4f9d75cf44aca92f651d71df8fbd] into tmpdir=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/.tmp, totalSize=36.5 K 2024-11-22T03:42:47,363 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] compactions.Compactor(225): Compacting 71419a78f75042e6893695fa66f0247a, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=249, earliestPutTs=1732246961696 2024-11-22T03:42:47,363 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] compactions.Compactor(225): Compacting 22511ae460cf4409ad7bfc2f0def223c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=271, earliestPutTs=1732246962861 2024-11-22T03:42:47,364 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] compactions.Compactor(225): Compacting a22a4f9d75cf44aca92f651d71df8fbd, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=290, earliestPutTs=1732246965035 2024-11-22T03:42:47,364 INFO [RS:0;c85114ed5096:42545-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8422287be02eb151a34ae7fe752afee0#A#compaction#475 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T03:42:47,365 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/.tmp/A/d19c8b3a6f5f4f7495ad59d9409b4d45 is 175, key is test_row_0/A:col10/1732246965035/Put/seqid=0 2024-11-22T03:42:47,369 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742390_1566 (size=31937) 2024-11-22T03:42:47,372 INFO [RS:0;c85114ed5096:42545-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8422287be02eb151a34ae7fe752afee0#C#compaction#477 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T03:42:47,373 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/.tmp/C/66697cf49fb942149cc8e7daafd35d06 is 50, key is test_row_0/C:col10/1732246965035/Put/seqid=0 2024-11-22T03:42:47,373 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/.tmp/A/d19c8b3a6f5f4f7495ad59d9409b4d45 as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/A/d19c8b3a6f5f4f7495ad59d9409b4d45 2024-11-22T03:42:47,376 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742391_1567 (size=12983) 2024-11-22T03:42:47,377 INFO [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8422287be02eb151a34ae7fe752afee0/A of 8422287be02eb151a34ae7fe752afee0 into d19c8b3a6f5f4f7495ad59d9409b4d45(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T03:42:47,377 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8422287be02eb151a34ae7fe752afee0: 2024-11-22T03:42:47,377 INFO [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., storeName=8422287be02eb151a34ae7fe752afee0/A, priority=13, startTime=1732246966938; duration=0sec 2024-11-22T03:42:47,377 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T03:42:47,377 DEBUG [RS:0;c85114ed5096:42545-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8422287be02eb151a34ae7fe752afee0:A 2024-11-22T03:42:47,379 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/.tmp/C/66697cf49fb942149cc8e7daafd35d06 as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/C/66697cf49fb942149cc8e7daafd35d06 2024-11-22T03:42:47,382 INFO [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8422287be02eb151a34ae7fe752afee0/C of 8422287be02eb151a34ae7fe752afee0 into 66697cf49fb942149cc8e7daafd35d06(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T03:42:47,382 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8422287be02eb151a34ae7fe752afee0: 2024-11-22T03:42:47,382 INFO [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., storeName=8422287be02eb151a34ae7fe752afee0/C, priority=13, startTime=1732246966938; duration=0sec 2024-11-22T03:42:47,382 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T03:42:47,382 DEBUG [RS:0;c85114ed5096:42545-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8422287be02eb151a34ae7fe752afee0:C 2024-11-22T03:42:47,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-22T03:42:47,499 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-22T03:42:47,512 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:42:47,518 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241122d33603a4211b4032a54b37a17f0b7246_8422287be02eb151a34ae7fe752afee0 to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241122d33603a4211b4032a54b37a17f0b7246_8422287be02eb151a34ae7fe752afee0 2024-11-22T03:42:47,520 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/.tmp/A/889bfadfdc494d568cf7630ba44e9e87, store: [table=TestAcidGuarantees family=A region=8422287be02eb151a34ae7fe752afee0] 2024-11-22T03:42:47,520 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/.tmp/A/889bfadfdc494d568cf7630ba44e9e87 is 175, key is test_row_0/A:col10/1732246965714/Put/seqid=0 2024-11-22T03:42:47,525 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742392_1568 (size=31255) 2024-11-22T03:42:47,860 INFO [Thread-2182 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2182. 2024-11-22T03:42:47,860 DEBUG [Thread-2182 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-22T03:42:47,860 DEBUG [Thread-2182 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T03:42:47,860 DEBUG [Thread-2182 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T03:42:47,860 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-22T03:42:47,879 INFO [Thread-2188 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2188. 2024-11-22T03:42:47,879 DEBUG [Thread-2188 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-22T03:42:47,879 DEBUG [Thread-2188 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T03:42:47,880 DEBUG [Thread-2188 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T03:42:47,880 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-22T03:42:47,880 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-22T03:42:47,880 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 17 2024-11-22T03:42:47,880 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 74 2024-11-22T03:42:47,880 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 13 2024-11-22T03:42:47,880 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 11 2024-11-22T03:42:47,880 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 133 2024-11-22T03:42:47,880 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-22T03:42:47,880 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4133 2024-11-22T03:42:47,880 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4108 2024-11-22T03:42:47,880 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4136 2024-11-22T03:42:47,880 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4143 2024-11-22T03:42:47,880 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4109 2024-11-22T03:42:47,880 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-22T03:42:47,880 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-22T03:42:47,880 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobGetAtomicity(AcidGuaranteesTestBase.java:122) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-22T03:42:47,880 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T03:42:47,880 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T03:42:47,880 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-22T03:42:47,881 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42627 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-22T03:42:47,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42627 {}] procedure2.ProcedureExecutor(1139): Stored pid=162, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-22T03:42:47,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-22T03:42:47,884 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732246967884"}]},"ts":"1732246967884"} 2024-11-22T03:42:47,885 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-22T03:42:47,885 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-22T03:42:47,886 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=163, ppid=162, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-22T03:42:47,887 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=164, ppid=163, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=8422287be02eb151a34ae7fe752afee0, UNASSIGN}] 2024-11-22T03:42:47,888 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=164, ppid=163, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=8422287be02eb151a34ae7fe752afee0, UNASSIGN 2024-11-22T03:42:47,888 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=164 updating hbase:meta row=8422287be02eb151a34ae7fe752afee0, regionState=CLOSING, regionLocation=c85114ed5096,42545,1732246786482 2024-11-22T03:42:47,890 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=164, ppid=163, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=8422287be02eb151a34ae7fe752afee0, UNASSIGN because future has completed 2024-11-22T03:42:47,890 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-22T03:42:47,890 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=165, ppid=164, state=RUNNABLE, hasLock=false; CloseRegionProcedure 8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482}] 2024-11-22T03:42:47,926 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=311, memsize=40.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/.tmp/A/889bfadfdc494d568cf7630ba44e9e87 2024-11-22T03:42:47,931 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/.tmp/B/294252a35e5d486db2224cb42753e806 is 50, key is test_row_0/B:col10/1732246965714/Put/seqid=0 2024-11-22T03:42:47,934 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742393_1569 (size=12301) 2024-11-22T03:42:47,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-22T03:42:48,043 INFO [RS_CLOSE_REGION-regionserver/c85114ed5096:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] handler.UnassignRegionHandler(122): Close 8422287be02eb151a34ae7fe752afee0 2024-11-22T03:42:48,043 DEBUG [RS_CLOSE_REGION-regionserver/c85114ed5096:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-22T03:42:48,043 DEBUG [RS_CLOSE_REGION-regionserver/c85114ed5096:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HRegion(1722): Closing 8422287be02eb151a34ae7fe752afee0, disabling compactions & flushes 2024-11-22T03:42:48,043 DEBUG [RS_CLOSE_REGION-regionserver/c85114ed5096:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HRegion(1993): waiting for 0 compactions & cache flush to complete for region TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0. 2024-11-22T03:42:48,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-22T03:42:48,335 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=311 (bloomFilter=true), to=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/.tmp/B/294252a35e5d486db2224cb42753e806 2024-11-22T03:42:48,342 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/.tmp/C/64a006525ee04355a22cbc31afef11b0 is 50, key is test_row_0/C:col10/1732246965714/Put/seqid=0 2024-11-22T03:42:48,357 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742394_1570 (size=12301) 2024-11-22T03:42:48,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-22T03:42:48,758 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=311 (bloomFilter=true), to=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/.tmp/C/64a006525ee04355a22cbc31afef11b0 2024-11-22T03:42:48,762 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/.tmp/A/889bfadfdc494d568cf7630ba44e9e87 as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/A/889bfadfdc494d568cf7630ba44e9e87 2024-11-22T03:42:48,765 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/A/889bfadfdc494d568cf7630ba44e9e87, entries=150, sequenceid=311, filesize=30.5 K 2024-11-22T03:42:48,766 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/.tmp/B/294252a35e5d486db2224cb42753e806 as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/B/294252a35e5d486db2224cb42753e806 2024-11-22T03:42:48,770 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/B/294252a35e5d486db2224cb42753e806, entries=150, sequenceid=311, filesize=12.0 K 2024-11-22T03:42:48,770 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/.tmp/C/64a006525ee04355a22cbc31afef11b0 as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/C/64a006525ee04355a22cbc31afef11b0 2024-11-22T03:42:48,775 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/C/64a006525ee04355a22cbc31afef11b0, entries=150, sequenceid=311, filesize=12.0 K 2024-11-22T03:42:48,776 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~317.11 KB/324720, currentSize=26.84 KB/27480 for 8422287be02eb151a34ae7fe752afee0 in 1675ms, sequenceid=311, compaction requested=false 2024-11-22T03:42:48,776 INFO [RS_CLOSE_REGION-regionserver/c85114ed5096:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0. 2024-11-22T03:42:48,776 DEBUG [RS_CLOSE_REGION-regionserver/c85114ed5096:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0. 2024-11-22T03:42:48,776 DEBUG [RS_CLOSE_REGION-regionserver/c85114ed5096:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0. after waiting 0 ms 2024-11-22T03:42:48,776 DEBUG [RS_CLOSE_REGION-regionserver/c85114ed5096:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0. 2024-11-22T03:42:48,776 INFO [RS_CLOSE_REGION-regionserver/c85114ed5096:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HRegion(2902): Flushing 8422287be02eb151a34ae7fe752afee0 3/3 column families, dataSize=26.84 KB heapSize=71.06 KB 2024-11-22T03:42:48,777 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8422287be02eb151a34ae7fe752afee0: 2024-11-22T03:42:48,784 DEBUG [RS_CLOSE_REGION-regionserver/c85114ed5096:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411224306ed84c9a7446d816e601f5d5ab5ff_8422287be02eb151a34ae7fe752afee0 is 50, key is test_row_0/A:col10/1732246967878/Put/seqid=0 2024-11-22T03:42:48,789 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742395_1571 (size=12454) 2024-11-22T03:42:48,790 DEBUG [RS_CLOSE_REGION-regionserver/c85114ed5096:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T03:42:48,796 INFO [RS_CLOSE_REGION-regionserver/c85114ed5096:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411224306ed84c9a7446d816e601f5d5ab5ff_8422287be02eb151a34ae7fe752afee0 to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411224306ed84c9a7446d816e601f5d5ab5ff_8422287be02eb151a34ae7fe752afee0 2024-11-22T03:42:48,797 DEBUG [RS_CLOSE_REGION-regionserver/c85114ed5096:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/.tmp/A/40d9a0b1b3d44ea487321dcc97e1bfb5, store: [table=TestAcidGuarantees family=A region=8422287be02eb151a34ae7fe752afee0] 2024-11-22T03:42:48,797 DEBUG [RS_CLOSE_REGION-regionserver/c85114ed5096:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/.tmp/A/40d9a0b1b3d44ea487321dcc97e1bfb5 is 175, key is test_row_0/A:col10/1732246967878/Put/seqid=0 2024-11-22T03:42:48,802 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742396_1572 (size=31255) 2024-11-22T03:42:49,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-22T03:42:49,203 INFO [RS_CLOSE_REGION-regionserver/c85114ed5096:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=321, memsize=8.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/.tmp/A/40d9a0b1b3d44ea487321dcc97e1bfb5 2024-11-22T03:42:49,208 DEBUG [RS_CLOSE_REGION-regionserver/c85114ed5096:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/.tmp/B/0d1abc5154194df78348ea6f1e9bdc6a is 50, key is test_row_0/B:col10/1732246967878/Put/seqid=0 2024-11-22T03:42:49,211 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742397_1573 (size=12301) 2024-11-22T03:42:49,612 INFO [RS_CLOSE_REGION-regionserver/c85114ed5096:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=321 (bloomFilter=true), to=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/.tmp/B/0d1abc5154194df78348ea6f1e9bdc6a 2024-11-22T03:42:49,618 DEBUG [RS_CLOSE_REGION-regionserver/c85114ed5096:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/.tmp/C/946a00ad786b4e6d9eb50f23bfeaf1ef is 50, key is test_row_0/C:col10/1732246967878/Put/seqid=0 2024-11-22T03:42:49,621 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742398_1574 (size=12301) 2024-11-22T03:42:50,022 INFO [RS_CLOSE_REGION-regionserver/c85114ed5096:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=321 (bloomFilter=true), to=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/.tmp/C/946a00ad786b4e6d9eb50f23bfeaf1ef 2024-11-22T03:42:50,025 DEBUG [RS_CLOSE_REGION-regionserver/c85114ed5096:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/.tmp/A/40d9a0b1b3d44ea487321dcc97e1bfb5 as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/A/40d9a0b1b3d44ea487321dcc97e1bfb5 2024-11-22T03:42:50,028 INFO [RS_CLOSE_REGION-regionserver/c85114ed5096:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/A/40d9a0b1b3d44ea487321dcc97e1bfb5, entries=150, sequenceid=321, filesize=30.5 K 2024-11-22T03:42:50,029 DEBUG [RS_CLOSE_REGION-regionserver/c85114ed5096:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/.tmp/B/0d1abc5154194df78348ea6f1e9bdc6a as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/B/0d1abc5154194df78348ea6f1e9bdc6a 2024-11-22T03:42:50,031 INFO [RS_CLOSE_REGION-regionserver/c85114ed5096:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/B/0d1abc5154194df78348ea6f1e9bdc6a, entries=150, sequenceid=321, filesize=12.0 K 2024-11-22T03:42:50,032 DEBUG [RS_CLOSE_REGION-regionserver/c85114ed5096:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/.tmp/C/946a00ad786b4e6d9eb50f23bfeaf1ef as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/C/946a00ad786b4e6d9eb50f23bfeaf1ef 2024-11-22T03:42:50,037 INFO [RS_CLOSE_REGION-regionserver/c85114ed5096:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/C/946a00ad786b4e6d9eb50f23bfeaf1ef, entries=150, sequenceid=321, filesize=12.0 K 2024-11-22T03:42:50,038 INFO [RS_CLOSE_REGION-regionserver/c85114ed5096:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~71.02 KB/72720, currentSize=0 B/0 for 8422287be02eb151a34ae7fe752afee0 in 1262ms, sequenceid=321, compaction requested=true 2024-11-22T03:42:50,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-22T03:42:50,038 DEBUG [StoreCloser-TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/A/2e8ad2f7b1f74bb9bf07028fb9d30310, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/A/d14c9c77ce1a494da6bfea3849d1f037, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/A/84c77362f3a8479ba4a6997ea26478a5, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/A/938c4314a44746aba24bca9f781823e2, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/A/f58c125a588e45e08433dff8062a916c, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/A/08d77b7752a94b26971efbb82978da3b, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/A/f3a5993a5b7a4d7baccc7a9b2f3d5aaf, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/A/991973affb45468ab95b18ed8152698d, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/A/1a92649cae8d4751903576d32badb85b, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/A/c3e84477a4504fc4bf2e7dba0c087cc5, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/A/13f7cb4e7374438d8bd0d585c0c38610, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/A/d0625ea7b4304705aabef95e0c1ecfe5, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/A/0720a04112f143309f37a2199a4a5f5a, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/A/013d0e162c6048a094d054d77649f0d6, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/A/90b9a6a483f24988990f2a8c6116c3b0, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/A/ed064f0700954704a3076922b772cdc9, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/A/0c138a3b9dd54e8d9caa9b80bc28ce10, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/A/02e7cbda247a49cc87aa17302630af4f, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/A/bce03a2cd14944ac891aa8baac4adbd5, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/A/aaa6f22fd13349bdae1afebf8ac400d5] to archive 2024-11-22T03:42:50,039 DEBUG [StoreCloser-TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-22T03:42:50,040 DEBUG [StoreCloser-TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/A/2e8ad2f7b1f74bb9bf07028fb9d30310 to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/A/2e8ad2f7b1f74bb9bf07028fb9d30310 2024-11-22T03:42:50,041 DEBUG [StoreCloser-TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/A/d14c9c77ce1a494da6bfea3849d1f037 to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/A/d14c9c77ce1a494da6bfea3849d1f037 2024-11-22T03:42:50,042 DEBUG [StoreCloser-TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/A/84c77362f3a8479ba4a6997ea26478a5 to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/A/84c77362f3a8479ba4a6997ea26478a5 2024-11-22T03:42:50,042 DEBUG [StoreCloser-TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/A/938c4314a44746aba24bca9f781823e2 to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/A/938c4314a44746aba24bca9f781823e2 2024-11-22T03:42:50,043 DEBUG [StoreCloser-TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/A/f58c125a588e45e08433dff8062a916c to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/A/f58c125a588e45e08433dff8062a916c 2024-11-22T03:42:50,044 DEBUG [StoreCloser-TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/A/08d77b7752a94b26971efbb82978da3b to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/A/08d77b7752a94b26971efbb82978da3b 2024-11-22T03:42:50,045 DEBUG [StoreCloser-TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/A/f3a5993a5b7a4d7baccc7a9b2f3d5aaf to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/A/f3a5993a5b7a4d7baccc7a9b2f3d5aaf 2024-11-22T03:42:50,045 DEBUG [StoreCloser-TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/A/991973affb45468ab95b18ed8152698d to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/A/991973affb45468ab95b18ed8152698d 2024-11-22T03:42:50,046 DEBUG [StoreCloser-TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/A/1a92649cae8d4751903576d32badb85b to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/A/1a92649cae8d4751903576d32badb85b 2024-11-22T03:42:50,047 DEBUG [StoreCloser-TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/A/c3e84477a4504fc4bf2e7dba0c087cc5 to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/A/c3e84477a4504fc4bf2e7dba0c087cc5 2024-11-22T03:42:50,047 DEBUG [StoreCloser-TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/A/13f7cb4e7374438d8bd0d585c0c38610 to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/A/13f7cb4e7374438d8bd0d585c0c38610 2024-11-22T03:42:50,048 DEBUG [StoreCloser-TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/A/d0625ea7b4304705aabef95e0c1ecfe5 to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/A/d0625ea7b4304705aabef95e0c1ecfe5 2024-11-22T03:42:50,049 DEBUG [StoreCloser-TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/A/0720a04112f143309f37a2199a4a5f5a to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/A/0720a04112f143309f37a2199a4a5f5a 2024-11-22T03:42:50,050 DEBUG [StoreCloser-TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/A/013d0e162c6048a094d054d77649f0d6 to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/A/013d0e162c6048a094d054d77649f0d6 2024-11-22T03:42:50,050 DEBUG [StoreCloser-TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/A/90b9a6a483f24988990f2a8c6116c3b0 to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/A/90b9a6a483f24988990f2a8c6116c3b0 2024-11-22T03:42:50,051 DEBUG [StoreCloser-TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/A/ed064f0700954704a3076922b772cdc9 to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/A/ed064f0700954704a3076922b772cdc9 2024-11-22T03:42:50,052 DEBUG [StoreCloser-TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/A/0c138a3b9dd54e8d9caa9b80bc28ce10 to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/A/0c138a3b9dd54e8d9caa9b80bc28ce10 2024-11-22T03:42:50,052 DEBUG [StoreCloser-TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/A/02e7cbda247a49cc87aa17302630af4f to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/A/02e7cbda247a49cc87aa17302630af4f 2024-11-22T03:42:50,053 DEBUG [StoreCloser-TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/A/bce03a2cd14944ac891aa8baac4adbd5 to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/A/bce03a2cd14944ac891aa8baac4adbd5 2024-11-22T03:42:50,054 DEBUG [StoreCloser-TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/A/aaa6f22fd13349bdae1afebf8ac400d5 to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/A/aaa6f22fd13349bdae1afebf8ac400d5 2024-11-22T03:42:50,055 DEBUG [StoreCloser-TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/B/bc9ddf9d6ddb4cadb7285a33c5637c76, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/B/4d255fc3ee3c47de92044e1ad0b0fac3, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/B/d85467796c174b3b9d6176079597e3cf, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/B/4104be0e041f43308af292e753ff368c, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/B/7c04c26321b94ba8b67a3c48d13e214e, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/B/276fa3c198bf43f0af4cdee49a168057, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/B/651411e40abe4a46987cf338155c610b, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/B/95ac750810af44f6b683b180c5c55c4d, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/B/477f1f9796664cde8f850858df092ebd, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/B/bf28dcbf9bea40f1a60b37230368093e, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/B/05ad277a790e445eb8957f5e67a78ee7, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/B/9a2d7fe083cf4333a06fb1455ffde479, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/B/0448d51157614cff8abf978e06e9494f, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/B/005ebc1f537144f38e392eb604d35bd0, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/B/6b3b676052d34707bccae42e76c62eb2, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/B/a172ebbd77234473bc738fce310a34cd, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/B/abe6c4acb5324a9dbb6c8c4faada4a10, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/B/6c491051f018435fa87469695882acca, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/B/245cfa1382974849bdd2e12f1e7d3a48, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/B/7e89c9e9c59142d8a597564f6b374f21] to archive 2024-11-22T03:42:50,056 DEBUG [StoreCloser-TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-22T03:42:50,057 DEBUG [StoreCloser-TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/B/bc9ddf9d6ddb4cadb7285a33c5637c76 to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/B/bc9ddf9d6ddb4cadb7285a33c5637c76 2024-11-22T03:42:50,058 DEBUG [StoreCloser-TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/B/4d255fc3ee3c47de92044e1ad0b0fac3 to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/B/4d255fc3ee3c47de92044e1ad0b0fac3 2024-11-22T03:42:50,059 DEBUG [StoreCloser-TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/B/d85467796c174b3b9d6176079597e3cf to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/B/d85467796c174b3b9d6176079597e3cf 2024-11-22T03:42:50,059 DEBUG [StoreCloser-TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/B/4104be0e041f43308af292e753ff368c to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/B/4104be0e041f43308af292e753ff368c 2024-11-22T03:42:50,060 DEBUG [StoreCloser-TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/B/7c04c26321b94ba8b67a3c48d13e214e to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/B/7c04c26321b94ba8b67a3c48d13e214e 2024-11-22T03:42:50,061 DEBUG [StoreCloser-TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/B/276fa3c198bf43f0af4cdee49a168057 to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/B/276fa3c198bf43f0af4cdee49a168057 2024-11-22T03:42:50,061 DEBUG [StoreCloser-TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/B/651411e40abe4a46987cf338155c610b to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/B/651411e40abe4a46987cf338155c610b 2024-11-22T03:42:50,062 DEBUG [StoreCloser-TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/B/95ac750810af44f6b683b180c5c55c4d to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/B/95ac750810af44f6b683b180c5c55c4d 2024-11-22T03:42:50,063 DEBUG [StoreCloser-TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/B/477f1f9796664cde8f850858df092ebd to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/B/477f1f9796664cde8f850858df092ebd 2024-11-22T03:42:50,063 DEBUG [StoreCloser-TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/B/bf28dcbf9bea40f1a60b37230368093e to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/B/bf28dcbf9bea40f1a60b37230368093e 2024-11-22T03:42:50,064 DEBUG [StoreCloser-TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/B/05ad277a790e445eb8957f5e67a78ee7 to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/B/05ad277a790e445eb8957f5e67a78ee7 2024-11-22T03:42:50,065 DEBUG [StoreCloser-TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/B/9a2d7fe083cf4333a06fb1455ffde479 to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/B/9a2d7fe083cf4333a06fb1455ffde479 2024-11-22T03:42:50,065 DEBUG [StoreCloser-TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/B/0448d51157614cff8abf978e06e9494f to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/B/0448d51157614cff8abf978e06e9494f 2024-11-22T03:42:50,066 DEBUG [StoreCloser-TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/B/005ebc1f537144f38e392eb604d35bd0 to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/B/005ebc1f537144f38e392eb604d35bd0 2024-11-22T03:42:50,067 DEBUG [StoreCloser-TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/B/6b3b676052d34707bccae42e76c62eb2 to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/B/6b3b676052d34707bccae42e76c62eb2 2024-11-22T03:42:50,067 DEBUG [StoreCloser-TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/B/a172ebbd77234473bc738fce310a34cd to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/B/a172ebbd77234473bc738fce310a34cd 2024-11-22T03:42:50,068 DEBUG [StoreCloser-TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/B/abe6c4acb5324a9dbb6c8c4faada4a10 to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/B/abe6c4acb5324a9dbb6c8c4faada4a10 2024-11-22T03:42:50,069 DEBUG [StoreCloser-TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/B/6c491051f018435fa87469695882acca to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/B/6c491051f018435fa87469695882acca 2024-11-22T03:42:50,069 DEBUG [StoreCloser-TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/B/245cfa1382974849bdd2e12f1e7d3a48 to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/B/245cfa1382974849bdd2e12f1e7d3a48 2024-11-22T03:42:50,070 DEBUG [StoreCloser-TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/B/7e89c9e9c59142d8a597564f6b374f21 to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/B/7e89c9e9c59142d8a597564f6b374f21 2024-11-22T03:42:50,071 DEBUG [StoreCloser-TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/C/9b847a0ddb6a411980cdc265c368207b, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/C/60c175ad58b343a8b05a41fd91fa939c, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/C/6a403ba20043415fa48677d75ee90ed6, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/C/913085e92be04ce0a3138e7bd619645d, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/C/f9056aac4dc740059dd3efb37f86d910, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/C/f370abdfa2174d93bdfd2318f0c74cb7, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/C/c23efac2b8274234a93b10b14a7cd17c, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/C/76340497696140b7b91858c99dc3f992, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/C/fc6202d01a1d4f38be8f5a1a5a04eda8, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/C/5b22cb77a3fd48f29c3e34d53305de40, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/C/848f862c06e847e683a59ec7e0c11d81, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/C/0e93261a48454456868712b724585666, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/C/b73ed0bbb8174881a95580ceb54bf62b, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/C/c3203268621e47afbd8fa349f53e846d, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/C/f2d89d2f42544991adb170025d27ad77, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/C/2d6531b5a63e444f83ceef4ec84ec6ad, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/C/71419a78f75042e6893695fa66f0247a, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/C/cda48792dad4403c97cf102f1aaee83f, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/C/22511ae460cf4409ad7bfc2f0def223c, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/C/a22a4f9d75cf44aca92f651d71df8fbd] to archive 2024-11-22T03:42:50,072 DEBUG [StoreCloser-TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-22T03:42:50,073 DEBUG [StoreCloser-TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/C/9b847a0ddb6a411980cdc265c368207b to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/C/9b847a0ddb6a411980cdc265c368207b 2024-11-22T03:42:50,073 DEBUG [StoreCloser-TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/C/60c175ad58b343a8b05a41fd91fa939c to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/C/60c175ad58b343a8b05a41fd91fa939c 2024-11-22T03:42:50,074 DEBUG [StoreCloser-TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/C/6a403ba20043415fa48677d75ee90ed6 to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/C/6a403ba20043415fa48677d75ee90ed6 2024-11-22T03:42:50,075 DEBUG [StoreCloser-TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/C/913085e92be04ce0a3138e7bd619645d to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/C/913085e92be04ce0a3138e7bd619645d 2024-11-22T03:42:50,075 DEBUG [StoreCloser-TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/C/f9056aac4dc740059dd3efb37f86d910 to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/C/f9056aac4dc740059dd3efb37f86d910 2024-11-22T03:42:50,076 DEBUG [StoreCloser-TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/C/f370abdfa2174d93bdfd2318f0c74cb7 to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/C/f370abdfa2174d93bdfd2318f0c74cb7 2024-11-22T03:42:50,077 DEBUG [StoreCloser-TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/C/c23efac2b8274234a93b10b14a7cd17c to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/C/c23efac2b8274234a93b10b14a7cd17c 2024-11-22T03:42:50,077 DEBUG [StoreCloser-TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/C/76340497696140b7b91858c99dc3f992 to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/C/76340497696140b7b91858c99dc3f992 2024-11-22T03:42:50,078 DEBUG [StoreCloser-TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/C/fc6202d01a1d4f38be8f5a1a5a04eda8 to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/C/fc6202d01a1d4f38be8f5a1a5a04eda8 2024-11-22T03:42:50,079 DEBUG [StoreCloser-TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/C/5b22cb77a3fd48f29c3e34d53305de40 to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/C/5b22cb77a3fd48f29c3e34d53305de40 2024-11-22T03:42:50,079 DEBUG [StoreCloser-TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/C/848f862c06e847e683a59ec7e0c11d81 to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/C/848f862c06e847e683a59ec7e0c11d81 2024-11-22T03:42:50,080 DEBUG [StoreCloser-TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/C/0e93261a48454456868712b724585666 to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/C/0e93261a48454456868712b724585666 2024-11-22T03:42:50,080 DEBUG [StoreCloser-TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/C/b73ed0bbb8174881a95580ceb54bf62b to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/C/b73ed0bbb8174881a95580ceb54bf62b 2024-11-22T03:42:50,081 DEBUG [StoreCloser-TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/C/c3203268621e47afbd8fa349f53e846d to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/C/c3203268621e47afbd8fa349f53e846d 2024-11-22T03:42:50,082 DEBUG [StoreCloser-TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/C/f2d89d2f42544991adb170025d27ad77 to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/C/f2d89d2f42544991adb170025d27ad77 2024-11-22T03:42:50,083 DEBUG [StoreCloser-TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/C/2d6531b5a63e444f83ceef4ec84ec6ad to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/C/2d6531b5a63e444f83ceef4ec84ec6ad 2024-11-22T03:42:50,083 DEBUG [StoreCloser-TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/C/71419a78f75042e6893695fa66f0247a to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/C/71419a78f75042e6893695fa66f0247a 2024-11-22T03:42:50,084 DEBUG [StoreCloser-TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/C/cda48792dad4403c97cf102f1aaee83f to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/C/cda48792dad4403c97cf102f1aaee83f 2024-11-22T03:42:50,085 DEBUG [StoreCloser-TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/C/22511ae460cf4409ad7bfc2f0def223c to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/C/22511ae460cf4409ad7bfc2f0def223c 2024-11-22T03:42:50,085 DEBUG [StoreCloser-TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/C/a22a4f9d75cf44aca92f651d71df8fbd to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/C/a22a4f9d75cf44aca92f651d71df8fbd 2024-11-22T03:42:50,088 DEBUG [RS_CLOSE_REGION-regionserver/c85114ed5096:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/recovered.edits/324.seqid, newMaxSeqId=324, maxSeqId=4 2024-11-22T03:42:50,089 INFO [RS_CLOSE_REGION-regionserver/c85114ed5096:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0. 2024-11-22T03:42:50,089 DEBUG [RS_CLOSE_REGION-regionserver/c85114ed5096:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HRegion(1676): Region close journal for 8422287be02eb151a34ae7fe752afee0: Waiting for close lock at 1732246968043Running coprocessor pre-close hooks at 1732246968043Disabling compacts and flushes for region at 1732246968043Disabling writes for close at 1732246968776 (+733 ms)Obtaining lock to block concurrent updates at 1732246968776Preparing flush snapshotting stores in 8422287be02eb151a34ae7fe752afee0 at 1732246968776Finished memstore snapshotting TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0., syncing WAL and waiting on mvcc, flushsize=dataSize=27480, getHeapSize=72720, getOffHeapSize=0, getCellsCount=600 at 1732246968777 (+1 ms)Flushing stores of TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0. at 1732246968777Flushing 8422287be02eb151a34ae7fe752afee0/A: creating writer at 1732246968777Flushing mob file 8422287be02eb151a34ae7fe752afee0/A: appending metadata at 1732246968784 (+7 ms)Flushing mob file 8422287be02eb151a34ae7fe752afee0/A: closing flushed file at 1732246968784Flushing 8422287be02eb151a34ae7fe752afee0/A: appending metadata at 1732246968797 (+13 ms)Flushing 8422287be02eb151a34ae7fe752afee0/A: closing flushed file at 1732246968797Flushing 8422287be02eb151a34ae7fe752afee0/B: creating writer at 1732246969206 (+409 ms)Flushing 8422287be02eb151a34ae7fe752afee0/B: appending metadata at 1732246969208 (+2 ms)Flushing 8422287be02eb151a34ae7fe752afee0/B: closing flushed file at 1732246969208Flushing 8422287be02eb151a34ae7fe752afee0/C: creating writer at 1732246969616 (+408 ms)Flushing 8422287be02eb151a34ae7fe752afee0/C: appending metadata at 1732246969618 (+2 ms)Flushing 8422287be02eb151a34ae7fe752afee0/C: closing flushed file at 1732246969618Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3e3bfa3c: reopening flushed file at 1732246970025 (+407 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@78d0a439: reopening flushed file at 1732246970028 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@59b5f646: reopening flushed file at 1732246970031 (+3 ms)Finished flush of dataSize ~26.84 KB/27480, heapSize ~71.02 KB/72720, currentSize=0 B/0 for 8422287be02eb151a34ae7fe752afee0 in 1262ms, sequenceid=321, compaction requested=true at 1732246970038 (+7 ms)Writing region close event to WAL at 1732246970086 (+48 ms)Running coprocessor post-close hooks at 1732246970089 (+3 ms)Closed at 1732246970089 2024-11-22T03:42:50,090 INFO [RS_CLOSE_REGION-regionserver/c85114ed5096:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] handler.UnassignRegionHandler(157): Closed 8422287be02eb151a34ae7fe752afee0 2024-11-22T03:42:50,091 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=164 updating hbase:meta row=8422287be02eb151a34ae7fe752afee0, regionState=CLOSED 2024-11-22T03:42:50,092 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=165, ppid=164, state=RUNNABLE, hasLock=false; CloseRegionProcedure 8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 because future has completed 2024-11-22T03:42:50,093 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=165, resume processing ppid=164 2024-11-22T03:42:50,094 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=165, ppid=164, state=SUCCESS, hasLock=false; CloseRegionProcedure 8422287be02eb151a34ae7fe752afee0, server=c85114ed5096,42545,1732246786482 in 2.2020 sec 2024-11-22T03:42:50,095 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=164, resume processing ppid=163 2024-11-22T03:42:50,095 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=164, ppid=163, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=8422287be02eb151a34ae7fe752afee0, UNASSIGN in 2.2070 sec 2024-11-22T03:42:50,096 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=163, resume processing ppid=162 2024-11-22T03:42:50,096 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=163, ppid=162, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 2.2090 sec 2024-11-22T03:42:50,097 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732246970097"}]},"ts":"1732246970097"} 2024-11-22T03:42:50,098 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-22T03:42:50,098 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-22T03:42:50,099 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=162, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 2.2170 sec 2024-11-22T03:42:52,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-22T03:42:52,059 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-22T03:42:52,059 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42627 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-22T03:42:52,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42627 {}] procedure2.ProcedureExecutor(1139): Stored pid=166, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-22T03:42:52,061 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=166, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-22T03:42:52,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-22T03:42:52,062 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=166, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-22T03:42:52,063 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0 2024-11-22T03:42:52,065 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/A, FileablePath, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/B, FileablePath, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/C, FileablePath, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/recovered.edits] 2024-11-22T03:42:52,067 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/A/40d9a0b1b3d44ea487321dcc97e1bfb5 to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/A/40d9a0b1b3d44ea487321dcc97e1bfb5 2024-11-22T03:42:52,068 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/A/889bfadfdc494d568cf7630ba44e9e87 to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/A/889bfadfdc494d568cf7630ba44e9e87 2024-11-22T03:42:52,069 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/A/d19c8b3a6f5f4f7495ad59d9409b4d45 to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/A/d19c8b3a6f5f4f7495ad59d9409b4d45 2024-11-22T03:42:52,071 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/B/0d1abc5154194df78348ea6f1e9bdc6a to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/B/0d1abc5154194df78348ea6f1e9bdc6a 2024-11-22T03:42:52,072 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/B/294252a35e5d486db2224cb42753e806 to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/B/294252a35e5d486db2224cb42753e806 2024-11-22T03:42:52,073 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/B/48989c538a2f433ba12f955260d8a700 to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/B/48989c538a2f433ba12f955260d8a700 2024-11-22T03:42:52,074 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/C/64a006525ee04355a22cbc31afef11b0 to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/C/64a006525ee04355a22cbc31afef11b0 2024-11-22T03:42:52,075 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/C/66697cf49fb942149cc8e7daafd35d06 to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/C/66697cf49fb942149cc8e7daafd35d06 2024-11-22T03:42:52,076 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/C/946a00ad786b4e6d9eb50f23bfeaf1ef to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/C/946a00ad786b4e6d9eb50f23bfeaf1ef 2024-11-22T03:42:52,078 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/recovered.edits/324.seqid to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0/recovered.edits/324.seqid 2024-11-22T03:42:52,078 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/default/TestAcidGuarantees/8422287be02eb151a34ae7fe752afee0 2024-11-22T03:42:52,078 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-22T03:42:52,079 DEBUG [PEWorker-2 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-22T03:42:52,079 DEBUG [PEWorker-2 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-22T03:42:52,081 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112203a74c5561ae41b49ba11e199b7c0f5f_8422287be02eb151a34ae7fe752afee0 to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112203a74c5561ae41b49ba11e199b7c0f5f_8422287be02eb151a34ae7fe752afee0 2024-11-22T03:42:52,084 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411222683af9cf0304abdbb554b88a9cbf243_8422287be02eb151a34ae7fe752afee0 to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411222683af9cf0304abdbb554b88a9cbf243_8422287be02eb151a34ae7fe752afee0 2024-11-22T03:42:52,085 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112229a410138d7a4528a77ee0fbb50e27c6_8422287be02eb151a34ae7fe752afee0 to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112229a410138d7a4528a77ee0fbb50e27c6_8422287be02eb151a34ae7fe752afee0 2024-11-22T03:42:52,086 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411223dade7110c8d442ba6a0816b290ae6e1_8422287be02eb151a34ae7fe752afee0 to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411223dade7110c8d442ba6a0816b290ae6e1_8422287be02eb151a34ae7fe752afee0 2024-11-22T03:42:52,086 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241122410292eb2ce4420dadfcca6799089867_8422287be02eb151a34ae7fe752afee0 to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241122410292eb2ce4420dadfcca6799089867_8422287be02eb151a34ae7fe752afee0 2024-11-22T03:42:52,087 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411224306ed84c9a7446d816e601f5d5ab5ff_8422287be02eb151a34ae7fe752afee0 to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411224306ed84c9a7446d816e601f5d5ab5ff_8422287be02eb151a34ae7fe752afee0 2024-11-22T03:42:52,088 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241122435db259e16b4ddd868a08fad804a3fa_8422287be02eb151a34ae7fe752afee0 to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241122435db259e16b4ddd868a08fad804a3fa_8422287be02eb151a34ae7fe752afee0 2024-11-22T03:42:52,088 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112243bdd4c1a8344ad7a9fe7f80ca24ead1_8422287be02eb151a34ae7fe752afee0 to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112243bdd4c1a8344ad7a9fe7f80ca24ead1_8422287be02eb151a34ae7fe752afee0 2024-11-22T03:42:52,089 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411224b4be423161c4a0584d2b5cf54a51719_8422287be02eb151a34ae7fe752afee0 to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411224b4be423161c4a0584d2b5cf54a51719_8422287be02eb151a34ae7fe752afee0 2024-11-22T03:42:52,090 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411226e41f0edd0a04841b36d952b4689f447_8422287be02eb151a34ae7fe752afee0 to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411226e41f0edd0a04841b36d952b4689f447_8422287be02eb151a34ae7fe752afee0 2024-11-22T03:42:52,090 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411227bb911e2d0cd4413a8a73683153faed0_8422287be02eb151a34ae7fe752afee0 to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411227bb911e2d0cd4413a8a73683153faed0_8422287be02eb151a34ae7fe752afee0 2024-11-22T03:42:52,091 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241122b1c73c919e654418adb41947c1fedffd_8422287be02eb151a34ae7fe752afee0 to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241122b1c73c919e654418adb41947c1fedffd_8422287be02eb151a34ae7fe752afee0 2024-11-22T03:42:52,092 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241122c44edc5f1abe4b6ea15b45c26929e141_8422287be02eb151a34ae7fe752afee0 to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241122c44edc5f1abe4b6ea15b45c26929e141_8422287be02eb151a34ae7fe752afee0 2024-11-22T03:42:52,092 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241122d33603a4211b4032a54b37a17f0b7246_8422287be02eb151a34ae7fe752afee0 to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241122d33603a4211b4032a54b37a17f0b7246_8422287be02eb151a34ae7fe752afee0 2024-11-22T03:42:52,093 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241122dfcbfb5b53974871ac7418c046dab708_8422287be02eb151a34ae7fe752afee0 to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241122dfcbfb5b53974871ac7418c046dab708_8422287be02eb151a34ae7fe752afee0 2024-11-22T03:42:52,094 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241122e6a3098f7b7947c4b0ec58ad09fef8fe_8422287be02eb151a34ae7fe752afee0 to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241122e6a3098f7b7947c4b0ec58ad09fef8fe_8422287be02eb151a34ae7fe752afee0 2024-11-22T03:42:52,095 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241122ed86cd200aeb4043b69dd85f13411b12_8422287be02eb151a34ae7fe752afee0 to hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241122ed86cd200aeb4043b69dd85f13411b12_8422287be02eb151a34ae7fe752afee0 2024-11-22T03:42:52,095 DEBUG [PEWorker-2 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-22T03:42:52,097 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=166, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-22T03:42:52,099 WARN [PEWorker-2 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-22T03:42:52,100 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-22T03:42:52,101 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=166, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-22T03:42:52,101 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-22T03:42:52,102 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732246972101"}]},"ts":"9223372036854775807"} 2024-11-22T03:42:52,104 INFO [PEWorker-2 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-22T03:42:52,104 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 8422287be02eb151a34ae7fe752afee0, NAME => 'TestAcidGuarantees,,1732246943404.8422287be02eb151a34ae7fe752afee0.', STARTKEY => '', ENDKEY => ''}] 2024-11-22T03:42:52,104 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-22T03:42:52,104 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732246972104"}]},"ts":"9223372036854775807"} 2024-11-22T03:42:52,107 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-22T03:42:52,107 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(133): Finished pid=166, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-22T03:42:52,108 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=166, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 48 msec 2024-11-22T03:42:52,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42627 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-22T03:42:52,169 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-22T03:42:52,169 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-22T03:42:52,176 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testMobGetAtomicity Thread=239 (was 238) - Thread LEAK? -, OpenFileDescriptor=447 (was 443) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=367 (was 365) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=3424 (was 3053) - AvailableMemoryMB LEAK? - 2024-11-22T03:42:52,176 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-22T03:42:52,176 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-22T03:42:52,176 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-22T03:42:52,176 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T03:42:52,176 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T03:42:52,176 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-22T03:42:52,177 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-22T03:42:52,177 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1284221299, stopped=false 2024-11-22T03:42:52,177 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=c85114ed5096,42627,1732246785796 2024-11-22T03:42:52,178 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42627-0x1006583a7f00000, quorum=127.0.0.1:64888, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-22T03:42:52,178 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42545-0x1006583a7f00001, quorum=127.0.0.1:64888, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-22T03:42:52,178 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42627-0x1006583a7f00000, quorum=127.0.0.1:64888, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-22T03:42:52,178 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42545-0x1006583a7f00001, quorum=127.0.0.1:64888, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-22T03:42:52,178 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-22T03:42:52,179 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-22T03:42:52,179 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-22T03:42:52,179 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T03:42:52,179 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:42545-0x1006583a7f00001, quorum=127.0.0.1:64888, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-22T03:42:52,179 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:42627-0x1006583a7f00000, quorum=127.0.0.1:64888, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-22T03:42:52,179 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'c85114ed5096,42545,1732246786482' ***** 2024-11-22T03:42:52,179 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-22T03:42:52,180 INFO [RS:0;c85114ed5096:42545 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-22T03:42:52,180 INFO [RS:0;c85114ed5096:42545 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-22T03:42:52,180 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-22T03:42:52,180 INFO [RS:0;c85114ed5096:42545 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-22T03:42:52,180 INFO [RS:0;c85114ed5096:42545 {}] regionserver.HRegionServer(959): stopping server c85114ed5096,42545,1732246786482 2024-11-22T03:42:52,180 INFO [RS:0;c85114ed5096:42545 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-22T03:42:52,180 INFO [RS:0;c85114ed5096:42545 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;c85114ed5096:42545. 2024-11-22T03:42:52,180 DEBUG [RS:0;c85114ed5096:42545 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-22T03:42:52,180 DEBUG [RS:0;c85114ed5096:42545 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T03:42:52,180 INFO [RS:0;c85114ed5096:42545 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-22T03:42:52,180 INFO [RS:0;c85114ed5096:42545 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-22T03:42:52,180 INFO [RS:0;c85114ed5096:42545 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-22T03:42:52,181 INFO [RS:0;c85114ed5096:42545 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-22T03:42:52,181 INFO [RS:0;c85114ed5096:42545 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-22T03:42:52,181 DEBUG [RS:0;c85114ed5096:42545 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-11-22T03:42:52,181 DEBUG [RS_CLOSE_META-regionserver/c85114ed5096:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-22T03:42:52,181 DEBUG [RS:0;c85114ed5096:42545 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-22T03:42:52,182 INFO [RS_CLOSE_META-regionserver/c85114ed5096:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-22T03:42:52,182 DEBUG [RS_CLOSE_META-regionserver/c85114ed5096:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-22T03:42:52,182 DEBUG [RS_CLOSE_META-regionserver/c85114ed5096:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-22T03:42:52,182 DEBUG [RS_CLOSE_META-regionserver/c85114ed5096:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-22T03:42:52,182 INFO [RS_CLOSE_META-regionserver/c85114ed5096:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=19.91 KB heapSize=35.20 KB 2024-11-22T03:42:52,200 DEBUG [RS_CLOSE_META-regionserver/c85114ed5096:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/hbase/meta/1588230740/.tmp/info/95d648d564524437911e1fab67cc3640 is 95, key is TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4./info:/1732246818158/DeleteFamily/seqid=0 2024-11-22T03:42:52,203 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742399_1575 (size=6514) 2024-11-22T03:42:52,246 INFO [regionserver/c85114ed5096:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-22T03:42:52,382 DEBUG [RS:0;c85114ed5096:42545 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-22T03:42:52,409 INFO [regionserver/c85114ed5096:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-22T03:42:52,410 INFO [regionserver/c85114ed5096:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-22T03:42:52,582 DEBUG [RS:0;c85114ed5096:42545 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-22T03:42:52,603 INFO [RS_CLOSE_META-regionserver/c85114ed5096:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=16.75 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/hbase/meta/1588230740/.tmp/info/95d648d564524437911e1fab67cc3640 2024-11-22T03:42:52,633 DEBUG [RS_CLOSE_META-regionserver/c85114ed5096:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/hbase/meta/1588230740/.tmp/ns/9f52e3fdad304efd901fee2f0088b563 is 93, key is TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4./ns:/1732246818111/DeleteFamily/seqid=0 2024-11-22T03:42:52,638 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742400_1576 (size=5906) 2024-11-22T03:42:52,782 DEBUG [RS:0;c85114ed5096:42545 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-22T03:42:52,982 DEBUG [RS:0;c85114ed5096:42545 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-22T03:42:53,039 INFO [RS_CLOSE_META-regionserver/c85114ed5096:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=608 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/hbase/meta/1588230740/.tmp/ns/9f52e3fdad304efd901fee2f0088b563 2024-11-22T03:42:53,055 DEBUG [RS_CLOSE_META-regionserver/c85114ed5096:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/hbase/meta/1588230740/.tmp/rep_barrier/acedbfd9e5d844afba5b4bf3203704d6 is 102, key is TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4./rep_barrier:/1732246818111/DeleteFamily/seqid=0 2024-11-22T03:42:53,058 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742401_1577 (size=6025) 2024-11-22T03:42:53,183 DEBUG [RS:0;c85114ed5096:42545 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-22T03:42:53,383 DEBUG [RS:0;c85114ed5096:42545 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-22T03:42:53,458 INFO [RS_CLOSE_META-regionserver/c85114ed5096:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=588 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/hbase/meta/1588230740/.tmp/rep_barrier/acedbfd9e5d844afba5b4bf3203704d6 2024-11-22T03:42:53,475 DEBUG [RS_CLOSE_META-regionserver/c85114ed5096:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/hbase/meta/1588230740/.tmp/table/c226dc7c94804080a12529dfa60838bc is 96, key is TestAcidGuarantees,,1732246790944.f970bd9f1e861e2b64a2fb87fb209ac4./table:/1732246818111/DeleteFamily/seqid=0 2024-11-22T03:42:53,478 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742402_1578 (size=5936) 2024-11-22T03:42:53,583 DEBUG [RS:0;c85114ed5096:42545 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-22T03:42:53,783 DEBUG [RS:0;c85114ed5096:42545 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-22T03:42:53,879 INFO [RS_CLOSE_META-regionserver/c85114ed5096:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.99 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/hbase/meta/1588230740/.tmp/table/c226dc7c94804080a12529dfa60838bc 2024-11-22T03:42:53,887 DEBUG [RS_CLOSE_META-regionserver/c85114ed5096:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/hbase/meta/1588230740/.tmp/info/95d648d564524437911e1fab67cc3640 as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/hbase/meta/1588230740/info/95d648d564524437911e1fab67cc3640 2024-11-22T03:42:53,891 INFO [RS_CLOSE_META-regionserver/c85114ed5096:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/hbase/meta/1588230740/info/95d648d564524437911e1fab67cc3640, entries=12, sequenceid=90, filesize=6.4 K 2024-11-22T03:42:53,892 DEBUG [RS_CLOSE_META-regionserver/c85114ed5096:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/hbase/meta/1588230740/.tmp/ns/9f52e3fdad304efd901fee2f0088b563 as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/hbase/meta/1588230740/ns/9f52e3fdad304efd901fee2f0088b563 2024-11-22T03:42:53,896 INFO [RS_CLOSE_META-regionserver/c85114ed5096:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/hbase/meta/1588230740/ns/9f52e3fdad304efd901fee2f0088b563, entries=8, sequenceid=90, filesize=5.8 K 2024-11-22T03:42:53,897 DEBUG [RS_CLOSE_META-regionserver/c85114ed5096:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/hbase/meta/1588230740/.tmp/rep_barrier/acedbfd9e5d844afba5b4bf3203704d6 as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/hbase/meta/1588230740/rep_barrier/acedbfd9e5d844afba5b4bf3203704d6 2024-11-22T03:42:53,900 INFO [RS_CLOSE_META-regionserver/c85114ed5096:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/hbase/meta/1588230740/rep_barrier/acedbfd9e5d844afba5b4bf3203704d6, entries=6, sequenceid=90, filesize=5.9 K 2024-11-22T03:42:53,900 DEBUG [RS_CLOSE_META-regionserver/c85114ed5096:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/hbase/meta/1588230740/.tmp/table/c226dc7c94804080a12529dfa60838bc as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/hbase/meta/1588230740/table/c226dc7c94804080a12529dfa60838bc 2024-11-22T03:42:53,903 INFO [RS_CLOSE_META-regionserver/c85114ed5096:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/hbase/meta/1588230740/table/c226dc7c94804080a12529dfa60838bc, entries=7, sequenceid=90, filesize=5.8 K 2024-11-22T03:42:53,904 INFO [RS_CLOSE_META-regionserver/c85114ed5096:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1722ms, sequenceid=90, compaction requested=false 2024-11-22T03:42:53,914 DEBUG [RS_CLOSE_META-regionserver/c85114ed5096:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/data/hbase/meta/1588230740/recovered.edits/93.seqid, newMaxSeqId=93, maxSeqId=1 2024-11-22T03:42:53,915 DEBUG [RS_CLOSE_META-regionserver/c85114ed5096:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-22T03:42:53,915 INFO [RS_CLOSE_META-regionserver/c85114ed5096:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-22T03:42:53,915 DEBUG [RS_CLOSE_META-regionserver/c85114ed5096:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732246972181Running coprocessor pre-close hooks at 1732246972181Disabling compacts and flushes for region at 1732246972181Disabling writes for close at 1732246972182 (+1 ms)Obtaining lock to block concurrent updates at 1732246972182Preparing flush snapshotting stores in 1588230740 at 1732246972182Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=20384, getHeapSize=35984, getOffHeapSize=0, getCellsCount=191 at 1732246972182Flushing stores of hbase:meta,,1.1588230740 at 1732246972182Flushing 1588230740/info: creating writer at 1732246972183 (+1 ms)Flushing 1588230740/info: appending metadata at 1732246972199 (+16 ms)Flushing 1588230740/info: closing flushed file at 1732246972199Flushing 1588230740/ns: creating writer at 1732246972607 (+408 ms)Flushing 1588230740/ns: appending metadata at 1732246972632 (+25 ms)Flushing 1588230740/ns: closing flushed file at 1732246972632Flushing 1588230740/rep_barrier: creating writer at 1732246973042 (+410 ms)Flushing 1588230740/rep_barrier: appending metadata at 1732246973055 (+13 ms)Flushing 1588230740/rep_barrier: closing flushed file at 1732246973055Flushing 1588230740/table: creating writer at 1732246973462 (+407 ms)Flushing 1588230740/table: appending metadata at 1732246973474 (+12 ms)Flushing 1588230740/table: closing flushed file at 1732246973474Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@74e2780f: reopening flushed file at 1732246973883 (+409 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@75f6742b: reopening flushed file at 1732246973891 (+8 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@212c7189: reopening flushed file at 1732246973896 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2b10e2c9: reopening flushed file at 1732246973900 (+4 ms)Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1722ms, sequenceid=90, compaction requested=false at 1732246973904 (+4 ms)Writing region close event to WAL at 1732246973910 (+6 ms)Running coprocessor post-close hooks at 1732246973915 (+5 ms)Closed at 1732246973915 2024-11-22T03:42:53,915 DEBUG [RS_CLOSE_META-regionserver/c85114ed5096:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-22T03:42:53,983 INFO [RS:0;c85114ed5096:42545 {}] regionserver.HRegionServer(976): stopping server c85114ed5096,42545,1732246786482; all regions closed. 2024-11-22T03:42:53,988 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073741834_1010 (size=25344) 2024-11-22T03:42:53,990 DEBUG [RS:0;c85114ed5096:42545 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/oldWALs 2024-11-22T03:42:53,990 INFO [RS:0;c85114ed5096:42545 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL c85114ed5096%2C42545%2C1732246786482.meta:.meta(num 1732246790492) 2024-11-22T03:42:53,992 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073741832_1008 (size=13193552) 2024-11-22T03:42:53,993 DEBUG [RS:0;c85114ed5096:42545 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/oldWALs 2024-11-22T03:42:53,993 INFO [RS:0;c85114ed5096:42545 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL c85114ed5096%2C42545%2C1732246786482:(num 1732246789575) 2024-11-22T03:42:53,994 DEBUG [RS:0;c85114ed5096:42545 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T03:42:53,994 INFO [RS:0;c85114ed5096:42545 {}] regionserver.LeaseManager(133): Closed leases 2024-11-22T03:42:53,994 INFO [RS:0;c85114ed5096:42545 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-22T03:42:53,994 INFO [RS:0;c85114ed5096:42545 {}] hbase.ChoreService(370): Chore service for: regionserver/c85114ed5096:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-22T03:42:53,994 INFO [RS:0;c85114ed5096:42545 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-22T03:42:53,994 INFO [regionserver/c85114ed5096:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-22T03:42:53,995 INFO [RS:0;c85114ed5096:42545 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:42545 2024-11-22T03:42:53,999 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42545-0x1006583a7f00001, quorum=127.0.0.1:64888, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/c85114ed5096,42545,1732246786482 2024-11-22T03:42:53,999 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42627-0x1006583a7f00000, quorum=127.0.0.1:64888, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-22T03:42:53,999 INFO [RS:0;c85114ed5096:42545 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-22T03:42:54,000 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [c85114ed5096,42545,1732246786482] 2024-11-22T03:42:54,001 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/c85114ed5096,42545,1732246786482 already deleted, retry=false 2024-11-22T03:42:54,001 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; c85114ed5096,42545,1732246786482 expired; onlineServers=0 2024-11-22T03:42:54,002 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master 'c85114ed5096,42627,1732246785796' ***** 2024-11-22T03:42:54,002 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-22T03:42:54,002 INFO [M:0;c85114ed5096:42627 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-22T03:42:54,002 INFO [M:0;c85114ed5096:42627 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-22T03:42:54,002 DEBUG [M:0;c85114ed5096:42627 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-22T03:42:54,002 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-22T03:42:54,002 DEBUG [M:0;c85114ed5096:42627 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-22T03:42:54,002 DEBUG [master/c85114ed5096:0:becomeActiveMaster-HFileCleaner.large.0-1732246789205 {}] cleaner.HFileCleaner(306): Exit Thread[master/c85114ed5096:0:becomeActiveMaster-HFileCleaner.large.0-1732246789205,5,FailOnTimeoutGroup] 2024-11-22T03:42:54,002 DEBUG [master/c85114ed5096:0:becomeActiveMaster-HFileCleaner.small.0-1732246789210 {}] cleaner.HFileCleaner(306): Exit Thread[master/c85114ed5096:0:becomeActiveMaster-HFileCleaner.small.0-1732246789210,5,FailOnTimeoutGroup] 2024-11-22T03:42:54,002 INFO [M:0;c85114ed5096:42627 {}] hbase.ChoreService(370): Chore service for: master/c85114ed5096:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-22T03:42:54,002 INFO [M:0;c85114ed5096:42627 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-22T03:42:54,002 DEBUG [M:0;c85114ed5096:42627 {}] master.HMaster(1795): Stopping service threads 2024-11-22T03:42:54,002 INFO [M:0;c85114ed5096:42627 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-22T03:42:54,003 INFO [M:0;c85114ed5096:42627 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-22T03:42:54,003 ERROR [M:0;c85114ed5096:42627 {}] procedure2.ProcedureExecutor(763): There are still active thread in group java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10], see STDOUT java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10] Thread[HFileArchiver-6,5,PEWorkerGroup] 2024-11-22T03:42:54,003 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42627-0x1006583a7f00000, quorum=127.0.0.1:64888, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-22T03:42:54,003 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42627-0x1006583a7f00000, quorum=127.0.0.1:64888, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-22T03:42:54,004 INFO [M:0;c85114ed5096:42627 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-22T03:42:54,004 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-22T03:42:54,004 DEBUG [M:0;c85114ed5096:42627 {}] zookeeper.ZKUtil(347): master:42627-0x1006583a7f00000, quorum=127.0.0.1:64888, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-22T03:42:54,004 WARN [M:0;c85114ed5096:42627 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-22T03:42:54,005 INFO [M:0;c85114ed5096:42627 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/.lastflushedseqids 2024-11-22T03:42:54,015 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742403_1579 (size=72) 2024-11-22T03:42:54,101 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42545-0x1006583a7f00001, quorum=127.0.0.1:64888, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-22T03:42:54,101 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42545-0x1006583a7f00001, quorum=127.0.0.1:64888, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-22T03:42:54,101 INFO [RS:0;c85114ed5096:42545 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-22T03:42:54,102 INFO [RS:0;c85114ed5096:42545 {}] regionserver.HRegionServer(1031): Exiting; stopping=c85114ed5096,42545,1732246786482; zookeeper connection closed. 2024-11-22T03:42:54,102 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@a15f59c {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@a15f59c 2024-11-22T03:42:54,103 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-11-22T03:42:54,416 INFO [M:0;c85114ed5096:42627 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-22T03:42:54,416 INFO [M:0;c85114ed5096:42627 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-22T03:42:54,416 DEBUG [M:0;c85114ed5096:42627 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-22T03:42:54,416 INFO [M:0;c85114ed5096:42627 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-22T03:42:54,416 DEBUG [M:0;c85114ed5096:42627 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-22T03:42:54,416 DEBUG [M:0;c85114ed5096:42627 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-22T03:42:54,416 DEBUG [M:0;c85114ed5096:42627 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-22T03:42:54,416 INFO [M:0;c85114ed5096:42627 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=727.12 KB heapSize=891.70 KB 2024-11-22T03:42:54,434 DEBUG [M:0;c85114ed5096:42627 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/b38c0cab0e254d3ca13ff8c6b33524af is 82, key is hbase:meta,,1/info:regioninfo/1732246790580/Put/seqid=0 2024-11-22T03:42:54,437 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742404_1580 (size=5672) 2024-11-22T03:42:54,837 INFO [M:0;c85114ed5096:42627 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=2024 (bloomFilter=true), to=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/b38c0cab0e254d3ca13ff8c6b33524af 2024-11-22T03:42:54,862 DEBUG [M:0;c85114ed5096:42627 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/3591869e2837418cb8a511173a46e7da is 2277, key is \x00\x00\x00\x00\x00\x00\x00\x91/proc:d/1732246946492/Put/seqid=0 2024-11-22T03:42:54,885 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742405_1581 (size=43496) 2024-11-22T03:42:54,885 INFO [M:0;c85114ed5096:42627 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=726.56 KB at sequenceid=2024 (bloomFilter=true), to=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/3591869e2837418cb8a511173a46e7da 2024-11-22T03:42:54,891 INFO [M:0;c85114ed5096:42627 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 3591869e2837418cb8a511173a46e7da 2024-11-22T03:42:54,913 DEBUG [M:0;c85114ed5096:42627 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/cab0150e0f484b09a4ac26bb5d00d374 is 69, key is c85114ed5096,42545,1732246786482/rs:state/1732246789336/Put/seqid=0 2024-11-22T03:42:54,933 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073742406_1582 (size=5156) 2024-11-22T03:42:55,335 INFO [M:0;c85114ed5096:42627 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=2024 (bloomFilter=true), to=hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/cab0150e0f484b09a4ac26bb5d00d374 2024-11-22T03:42:55,344 DEBUG [M:0;c85114ed5096:42627 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/b38c0cab0e254d3ca13ff8c6b33524af as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/b38c0cab0e254d3ca13ff8c6b33524af 2024-11-22T03:42:55,348 INFO [M:0;c85114ed5096:42627 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/b38c0cab0e254d3ca13ff8c6b33524af, entries=8, sequenceid=2024, filesize=5.5 K 2024-11-22T03:42:55,349 DEBUG [M:0;c85114ed5096:42627 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/3591869e2837418cb8a511173a46e7da as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/3591869e2837418cb8a511173a46e7da 2024-11-22T03:42:55,351 INFO [M:0;c85114ed5096:42627 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 3591869e2837418cb8a511173a46e7da 2024-11-22T03:42:55,352 INFO [M:0;c85114ed5096:42627 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/3591869e2837418cb8a511173a46e7da, entries=166, sequenceid=2024, filesize=42.5 K 2024-11-22T03:42:55,352 DEBUG [M:0;c85114ed5096:42627 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/cab0150e0f484b09a4ac26bb5d00d374 as hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/cab0150e0f484b09a4ac26bb5d00d374 2024-11-22T03:42:55,355 INFO [M:0;c85114ed5096:42627 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35439/user/jenkins/test-data/24d0e080-c1c9-0c19-ec2c-ba3e7289634a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/cab0150e0f484b09a4ac26bb5d00d374, entries=1, sequenceid=2024, filesize=5.0 K 2024-11-22T03:42:55,355 INFO [M:0;c85114ed5096:42627 {}] regionserver.HRegion(3140): Finished flush of dataSize ~727.12 KB/744570, heapSize ~891.40 KB/912792, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 939ms, sequenceid=2024, compaction requested=false 2024-11-22T03:42:55,356 INFO [M:0;c85114ed5096:42627 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-22T03:42:55,356 DEBUG [M:0;c85114ed5096:42627 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732246974416Disabling compacts and flushes for region at 1732246974416Disabling writes for close at 1732246974416Obtaining lock to block concurrent updates at 1732246974416Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1732246974416Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=744570, getHeapSize=913032, getOffHeapSize=0, getCellsCount=2215 at 1732246974417 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1732246974417Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1732246974417Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1732246974434 (+17 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1732246974434Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1732246974840 (+406 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1732246974862 (+22 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1732246974862Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1732246974891 (+29 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1732246974912 (+21 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1732246974912Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@118d6e5a: reopening flushed file at 1732246975342 (+430 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@362f92d5: reopening flushed file at 1732246975348 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@689ca6ae: reopening flushed file at 1732246975352 (+4 ms)Finished flush of dataSize ~727.12 KB/744570, heapSize ~891.40 KB/912792, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 939ms, sequenceid=2024, compaction requested=false at 1732246975355 (+3 ms)Writing region close event to WAL at 1732246975356 (+1 ms)Closed at 1732246975356 2024-11-22T03:42:55,358 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41449 is added to blk_1073741830_1006 (size=877313) 2024-11-22T03:42:55,359 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-22T03:42:55,359 INFO [M:0;c85114ed5096:42627 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-22T03:42:55,359 INFO [M:0;c85114ed5096:42627 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:42627 2024-11-22T03:42:55,359 INFO [M:0;c85114ed5096:42627 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-22T03:42:55,461 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42627-0x1006583a7f00000, quorum=127.0.0.1:64888, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-22T03:42:55,461 INFO [M:0;c85114ed5096:42627 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-22T03:42:55,461 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42627-0x1006583a7f00000, quorum=127.0.0.1:64888, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-22T03:42:55,469 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4269bd31{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-22T03:42:55,473 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@353d64f2{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-22T03:42:55,473 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-22T03:42:55,474 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@435f9f96{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-22T03:42:55,474 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@4fd7fa7c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ef424767-da8d-bd68-b32d-b1c48514e6de/hadoop.log.dir/,STOPPED} 2024-11-22T03:42:55,476 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-22T03:42:55,476 WARN [BP-418171142-172.17.0.2-1732246782681 heartbeating to localhost/127.0.0.1:35439 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-22T03:42:55,477 WARN [BP-418171142-172.17.0.2-1732246782681 heartbeating to localhost/127.0.0.1:35439 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-418171142-172.17.0.2-1732246782681 (Datanode Uuid 1704e707-0098-49a6-9504-b28a4f653dc9) service to localhost/127.0.0.1:35439 2024-11-22T03:42:55,477 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-22T03:42:55,479 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ef424767-da8d-bd68-b32d-b1c48514e6de/cluster_b5649d9b-69fe-21c7-9e96-7cb2331ed67f/data/data1/current/BP-418171142-172.17.0.2-1732246782681 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-22T03:42:55,480 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ef424767-da8d-bd68-b32d-b1c48514e6de/cluster_b5649d9b-69fe-21c7-9e96-7cb2331ed67f/data/data2/current/BP-418171142-172.17.0.2-1732246782681 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-22T03:42:55,480 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-22T03:42:55,488 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4ded99d3{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-22T03:42:55,488 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@557285fa{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-22T03:42:55,488 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-22T03:42:55,489 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@93c62df{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-22T03:42:55,489 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6b23711f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ef424767-da8d-bd68-b32d-b1c48514e6de/hadoop.log.dir/,STOPPED} 2024-11-22T03:42:55,506 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-22T03:42:55,603 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-22T03:42:55,605 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-22T03:42:55,626 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down